diff options
331 files changed, 8018 insertions, 2425 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd1f52692a..831f4df56c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -309,7 +309,7 @@ jobs: if: needs.ci-config.outputs.enabled == 'yes' env: jobname: StaticAnalysis - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - run: ci/install-dependencies.sh diff --git a/.gitignore b/.gitignore index 42fd7253b4..80b530bbed 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ /git-cvsimport /git-cvsserver /git-daemon +/git-diagnose /git-diff /git-diff-files /git-diff-index diff --git a/Documentation/Makefile b/Documentation/Makefile index 4f801f4e4c..bd6b6fcb93 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -24,10 +24,21 @@ MAN1_TXT += gitweb.txt # man5 / man7 guides (note: new guides should also be added to command-list.txt) MAN5_TXT += gitattributes.txt +MAN5_TXT += gitformat-bundle.txt +MAN5_TXT += gitformat-chunk.txt +MAN5_TXT += gitformat-commit-graph.txt +MAN5_TXT += gitformat-index.txt +MAN5_TXT += gitformat-pack.txt +MAN5_TXT += gitformat-signature.txt MAN5_TXT += githooks.txt MAN5_TXT += gitignore.txt MAN5_TXT += gitmailmap.txt MAN5_TXT += gitmodules.txt +MAN5_TXT += gitprotocol-capabilities.txt +MAN5_TXT += gitprotocol-common.txt +MAN5_TXT += gitprotocol-http.txt +MAN5_TXT += gitprotocol-pack.txt +MAN5_TXT += gitprotocol-v2.txt MAN5_TXT += gitrepository-layout.txt MAN5_TXT += gitweb.conf.txt @@ -95,26 +106,17 @@ TECH_DOCS += MyFirstObjectWalk TECH_DOCS += SubmittingPatches TECH_DOCS += ToolsForGit TECH_DOCS += technical/bitmap-format -TECH_DOCS += technical/bundle-format -TECH_DOCS += technical/cruft-packs +TECH_DOCS += technical/bundle-uri TECH_DOCS += technical/hash-function-transition -TECH_DOCS += technical/http-protocol -TECH_DOCS += technical/index-format TECH_DOCS += technical/long-running-process-protocol TECH_DOCS += technical/multi-pack-index -TECH_DOCS += technical/pack-format TECH_DOCS += technical/pack-heuristics -TECH_DOCS += technical/pack-protocol TECH_DOCS += technical/parallel-checkout TECH_DOCS += technical/partial-clone -TECH_DOCS += technical/protocol-capabilities -TECH_DOCS += technical/protocol-common -TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow -TECH_DOCS += technical/signature-format TECH_DOCS += technical/trivial-merge SP_ARTICLES += $(TECH_DOCS) SP_ARTICLES += technical/api-index @@ -290,6 +292,8 @@ cmds_txt = cmds-ancillaryinterrogators.txt \ cmds-synchingrepositories.txt \ cmds-synchelpers.txt \ cmds-guide.txt \ + cmds-developerinterfaces.txt \ + cmds-userinterfaces.txt \ cmds-purehelpers.txt \ cmds-foreignscminterface.txt diff --git a/Documentation/RelNotes/2.37.2.txt b/Documentation/RelNotes/2.37.2.txt new file mode 100644 index 0000000000..d82b29e014 --- /dev/null +++ b/Documentation/RelNotes/2.37.2.txt @@ -0,0 +1,88 @@ +Git 2.37.2 Release Notes +======================== + +This primarily is to backport various fixes accumulated on the 'master' +front since 2.37.1. + +Fixes since v2.37.1 +------------------- + + * "git shortlog -n" relied on the underlying qsort() to be stable, + which shouldn't have. Fixed. + + * Variable quoting fix in the vimdiff driver of "git mergetool". + + * An earlier attempt to plug leaks placed a clean-up label to jump to + at a bogus place, which as been corrected. + + * Fixes a long-standing corner case bug around directory renames in + the merge-ort strategy. + + * Recent update to vimdiff layout code has been made more robust + against different end-user vim settings. + + * In a non-bare repository, the behavior of Git when the + core.worktree configuration variable points at a directory that has + a repository as its subdirectory, regressed in Git 2.27 days. + + * References to commands-to-be-typed-literally in "git rebase" + documentation mark-up have been corrected. + + * Give _() markings to fatal/warning/usage: labels that are shown in + front of these messages. + + * "git mktree --missing" lazily fetched objects that are missing from + the local object store, which was totally unnecessary for the purpose + of creating the tree object(s) from its input. + + * Fixes for tests when the source directory has unusual characters in + its path, e.g. whitespaces, double-quotes, etc. + + * Adjust technical/bitmap-format to be formatted by AsciiDoc, and + add some missing information to the documentation. + + * Certain diff options are currently ignored when combined-diff is + shown; mark them as incompatible with the feature. + + * "git clone" from a repository with some ref whose HEAD is unborn + did not set the HEAD in the resulting repository correctly, which + has been corrected. + + * mkstemp() emulation on Windows has been improved. + + * Add missing documentation for "include" and "includeIf" features in + "git config" file format, which incidentally teaches the command + line completion to include them in its offerings. + + * Avoid "white/black-list" in documentation and code comments. + + * Workaround for a compiler warning against use of die() in + osx-keychain (in contrib/). + + * Workaround for a false positive compiler warning. + + * The resolve-undo information in the index was not protected against + GC, which has been corrected. + + * A corner case bug where lazily fetching objects from a promisor + remote resulted in infinite recursion has been corrected. + + * "git p4" working on UTF-16 files on Windows did not implement + CRLF-to-LF conversion correctly, which has been corrected. + + * "git p4" did not handle non-ASCII client name well, which has been + corrected. + + * "rerere-train" script (in contrib/) used to honor commit.gpgSign + while recreating the throw-away merges. + + * "git checkout" miscounted the paths it updated, which has been + corrected. + + * Fix for a bug that makes write-tree to fail to write out a + non-existent index as a tree, introduced in 2.37. + + * There was a bug in the codepath to upgrade generation information + in commit-graph from v1 to v2 format, which has been corrected. + +Also contains minor documentation updates and code clean-ups. diff --git a/Documentation/RelNotes/2.37.3.txt b/Documentation/RelNotes/2.37.3.txt new file mode 100644 index 0000000000..d66689e598 --- /dev/null +++ b/Documentation/RelNotes/2.37.3.txt @@ -0,0 +1,46 @@ +Git 2.37.3 Release Notes +======================== + +This primarily is to backport various fixes accumulated on the 'master' +front since 2.37.2. + +Fixes since v2.37.2 +------------------- + + * The build procedure for Windows that uses CMake has been updated to + pick up the shell interpreter from local installation location. + + * Conditionally allow building Python interpreter on Windows + + * Fix to lstat() emulation on Windows. + + * Older gcc with -Wall complains about the universal zero initializer + "struct s = { 0 };" idiom, which makes developers' lives + inconvenient (as -Werror is enabled by DEVELOPER=YesPlease). The + build procedure has been tweaked to help these compilers. + + * Plug memory leaks in the failure code path in the "merge-ort" merge + strategy backend. + + * Avoid repeatedly running getconf to ask libc version in the test + suite, and instead just as it once per script. + + * Platform-specific code that determines if a directory is OK to use + as a repository has been taught to report more details, especially + on Windows. + + * "vimdiff3" regression has been corrected. + + * "git fsck" reads mode from tree objects but canonicalizes the mode + before passing it to the logic to check object sanity, which has + hid broken tree objects from the checking logic. This has been + corrected, but to help exiting projects with broken tree objects + that they cannot fix retroactively, the severity of anomalies this + code detects has been demoted to "info" for now. + + * Fixes to sparse index compatibility work for "reset" and "checkout" + commands. + + * Documentation for "git add --renormalize" has been improved. + +Also contains other minor documentation updates and code clean-ups. diff --git a/Documentation/RelNotes/2.38.0.txt b/Documentation/RelNotes/2.38.0.txt index c394ec731e..91a822e62b 100644 --- a/Documentation/RelNotes/2.38.0.txt +++ b/Documentation/RelNotes/2.38.0.txt @@ -30,6 +30,48 @@ UI, Workflows & Features * Introduce a discovery.barerepository configuration variable that allows users to forbid discovery of bare repositories. + * Various messages that come from the pack-bitmap codepaths have been + tweaked. + + * "git rebase -i" learns to update branches whose tip appear in the + rebased range with "--update-refs" option. + + * "git ls-files" learns the "--format" option to tweak its output. + + * "git cat-file" learned an option to use the mailmap when showing + commit and tag objects. + + * When "git merge" finds that it cannot perform a merge, it should + restore the working tree to the state before the command was + initiated, but in some corner cases it didn't. + + * Operating modes like "--batch" of "git cat-file" command learned to + take NUL-terminated input, instead of one-item-per-line. + + * "git rm" has become more aware of the sparse-index feature. + + * "git rev-list --disk-usage" learned to take an optional value + "human" to show the reported value in human-readable format, like + "3.40MiB". + + * The "diagnose" feature to create a zip archive for diagnostic + material has been lifted from "scalar" and made into a feature of + "git bugreport". + + * The namespaces used by "log --decorate" from "refs/" hierarchy by + default has been tightened. + + * "git rev-list --ancestry-path=C A..B" is a natural extension of + "git rev-list A..B"; instead of choosing a subset of A..B to those + that have ancestry relationship with A, it lets a subset with + ancestry relationship with C. + + * "scalar" now enables built-in fsmonitor on enlisted repositories, + when able. + + * The bash prompt (in contrib/) learned to optionally indicate when + the index is unmerged. + Performance, Internal Implementation, Development Support etc. @@ -75,6 +117,28 @@ Performance, Internal Implementation, Development Support etc. * A coccinelle rule (in contrib/) to encourage use of COPY_ARRAY macro has been improved. + * API tweak to make it easier to run fuzz testing on commit-graph parser. + + * Omit fsync-related trace2 entries when their values are all zero. + + * The codepath to write multi-pack index has been taught to release a + large chunk of memory that holds an array of objects in the packs, + as soon as it is done with the array, to reduce memory consumption. + + * Add a level of redirection to array allocation API in xdiff part, + to make it easier to share with the libgit2 project. + + * "git fetch" client logs the partial clone filter used in the trace2 + output. + + * The "bundle URI" design gets documented. + + * The common ancestor negotiation exchange during a "git fetch" + session now leaves trace log. + + * Test portability improvements. + (merge 4d1d843be7 mt/rot13-in-c later to maint). + Fixes since v2.37 ----------------- @@ -84,85 +148,153 @@ Fixes since v2.37 * Certain diff options are currently ignored when combined-diff is shown; mark them as incompatible with the feature. - (merge cfb19ae05f rs/combine-diff-with-incompatible-options later to maint). * Adjust technical/bitmap-format to be formatted by AsciiDoc, and add some missing information to the documentation. - (merge ac7667bd44 ac/bitmap-format-doc later to maint). * Fixes for tests when the source directory has unusual characters in its path, e.g. whitespaces, double-quotes, etc. - (merge eb1cd60290 ab/test-quoting-fix later to maint). * "git mktree --missing" lazily fetched objects that are missing from the local object store, which was totally unnecessary for the purpose of creating the tree object(s) from its input. - (merge 817b0f6027 ro/mktree-allow-missing-fix later to maint). * Give _() markings to fatal/warning/usage: labels that are shown in front of these messages. - (merge a1fd2cf8cd dr/i18n-die-warn-error-usage later to maint). * References to commands-to-be-typed-literally in "git rebase" documentation mark-up have been corrected. - (merge 54e51e559e ds/git-rebase-doc-markup later to maint). * In a non-bare repository, the behavior of Git when the core.worktree configuration variable points at a directory that has a repository as its subdirectory, regressed in Git 2.27 days. - (merge d6c9a71755 gg/worktree-from-the-above later to maint). * Recent update to vimdiff layout code has been made more robust against different end-user vim settings. - (merge f3d7623a13 fr/vimdiff-layout-fix later to maint). - - * Plug various memory leaks. - (merge ece3974ba6 ab/leakfix later to maint). - * Plug various memory leaks in test-tool commands. - (merge f40a693450 ab/test-tool-leakfix later to maint). + * Plug various memory leaks, both in the main code and in test-tool + commands. * Fixes a long-standing corner case bug around directory renames in the merge-ort strategy. - (merge 751e165424 en/merge-dual-dir-renames-fix later to maint). * The resolve-undo information in the index was not protected against GC, which has been corrected. - (merge e0ad13977a jc/resolve-undo later to maint). * A corner case bug where lazily fetching objects from a promisor remote resulted in infinite recursion has been corrected. - (merge cb88b37cb9 hx/lookup-commit-in-graph-fix later to maint). * "git clone" from a repository with some ref whose HEAD is unborn did not set the HEAD in the resulting repository correctly, which has been corrected. - (merge daf7898abb jk/clone-unborn-confusion later to maint). * An earlier attempt to plug leaks placed a clean-up label to jump to at a bogus place, which as been corrected. - (merge 04393ae7f7 jk/diff-files-cleanup-fix later to maint). * Variable quoting fix in the vimdiff driver of "git mergetool" - (merge ccc7b5148b js/vimdiff-quotepath-fix later to maint). * "git shortlog -n" relied on the underlying qsort() to be stable, which shouldn't have. Fixed. - (merge df534dcbaa js/shortlog-sort-stably later to maint). * A fix for a regression in test framework. - (merge 7253f7ca9f js/ci-github-workflow-markup later to maint). * mkstemp() emulation on Windows has been improved. - (merge ae25974de3 rs/mingw-tighten-mkstemp later to maint). + + * Add missing documentation for "include" and "includeIf" features in + "git config" file format, which incidentally teaches the command + line completion to include them in its offerings. + + * Avoid "white/black-list" in documentation and code comments. + + * Workaround for a compiler warning against use of die() in + osx-keychain (in contrib/). + + * Workaround for a false positive compiler warning. + + * "git p4" working on UTF-16 files on Windows did not implement + CRLF-to-LF conversion correctly, which has been corrected. + + * "git p4" did not handle non-ASCII client name well, which has been + corrected. + + * "rerere-train" script (in contrib/) used to honor commit.gpgSign + while recreating the throw-away merges. + + * "git checkout" miscounted the paths it updated, which has been + corrected. + + * Fix for a bug that makes write-tree to fail to write out a + non-existent index as a tree, introduced in 2.37. + + * There was a bug in the codepath to upgrade generation information + in commit-graph from v1 to v2 format, which has been corrected. + + * Gitweb had legacy URL shortener that is specific to the way + projects hosted on kernel.org used to (but no longer) work, which + has been removed. + + * Fix build procedure for Windows that uses CMake so that it can pick + up the shell interpreter from local installation location. + + * Conditionally allow building Python interpreter on Windows + + * Fix to lstat() emulation on Windows. + + * Older gcc with -Wall complains about the universal zero initializer + "struct s = { 0 };" idiom, which makes developers' lives + inconvenient (as -Werror is enabled by DEVELOPER=YesPlease). The + build procedure has been tweaked to help these compilers. + + * Plug memory leaks in the failure code path in the "merge-ort" merge + strategy backend. + + * "git symbolic-ref symref non..sen..se" is now diagnosed as an error. + + * A follow-up fix to a fix for a regression in 2.36 around hooks. + + * Avoid repeatedly running getconf to ask libc version in the test + suite, and instead just as it once per script. + + * Platform-specific code that determines if a directory is OK to use + as a repository has been taught to report more details, especially + on Windows. + + * "vimdiff3" regression fix. + + * "git fsck" reads mode from tree objects but canonicalizes the mode + before passing it to the logic to check object sanity, which has + hid broken tree objects from the checking logic. This has been + corrected, but to help exiting projects with broken tree objects + that they cannot fix retroactively, the severity of anomalies this + code detects has been demoted to "info" for now. + + * Fixes to sparse index compatibility work for "reset" and "checkout" + commands. + + * An earlier optimization discarded a tree-object buffer that is + still in use, which has been corrected. + (merge 1490d7d82d jk/is-promisor-object-keep-tree-in-use later to maint). + + * Fix deadlocks between main Git process and subprocess spawned via + the pipe_command() API, that can kill "git add -p" that was + reimplemented in C recently. + (merge 716c1f649e jk/pipe-command-nonblock later to maint). + + * The sequencer machinery translated messages left in the reflog by + mistake, which has been corrected. + + * xcalloc(), imitating calloc(), takes "number of elements of the + array", and "size of a single element", in this order. A call that + does not follow this ordering has been corrected. + (merge c4bbd9bb8f sg/xcalloc-cocci-fix later to maint). + + * The preload-index codepath made copies of pathspec to give to + multiple threads, which were left leaked. + (merge 23578904da ad/preload-plug-memleak later to maint). + + * Update the version of Ubuntu used for GitHub Actions CI from 18.04 + to 22.04. + (merge ef46584831 ds/github-actions-use-newer-ubuntu later to maint). * Other code cleanup, docfix, build fix, etc. - (merge 5fd9d1738e jk/revisions-doc-markup-fix later to maint). - (merge 1971510c35 pb/diff-doc-raw-format later to maint). - (merge a3ba4fa715 cr/setup-bug-typo later to maint). - (merge 85845580d9 ds/vscode-settings later to maint). - (merge ec2f6c0cca en/t6429-test-must-be-empty-fix later to maint). - (merge 14deb585fb tb/pack-objects-remove-pahole-comment later to maint). - (merge 2ffb7d13ee ds/t5510-brokequote later to maint). - (merge 18337d406f ll/ls-files-tests-update later to maint). - (merge 8cdab69d96 tk/rev-parse-doc-clarify-at-u later to maint). + (merge 77b9e85c0f vd/fix-perf-tests later to maint).
\ No newline at end of file diff --git a/Documentation/config.txt b/Documentation/config.txt index e376d547ce..5b5b976569 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -445,6 +445,8 @@ include::config/i18n.txt[] include::config/imap.txt[] +include::config/includeif.txt[] + include::config/index.txt[] include::config/init.txt[] diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 3ea3124f7f..37afbaf5a4 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -722,8 +722,8 @@ core.sparseCheckout:: core.sparseCheckoutCone:: Enables the "cone mode" of the sparse checkout feature. When the sparse-checkout file contains a limited set of patterns, this - mode provides significant performance advantages. The "non - cone mode" can be requested to allow specifying a more flexible + mode provides significant performance advantages. The "non-cone + mode" can be requested to allow specifying more flexible patterns by setting this variable to 'false'. See linkgit:git-sparse-checkout[1] for more information. diff --git a/Documentation/config/includeif.txt b/Documentation/config/includeif.txt new file mode 100644 index 0000000000..82fe431c34 --- /dev/null +++ b/Documentation/config/includeif.txt @@ -0,0 +1,6 @@ +include.path:: +includeIf.<condition>.path:: + Special variables to include other configuration files. See + the "CONFIGURATION FILE" section in the main + linkgit:git-config[1] documentation, + specifically the "Includes" and "Conditional Includes" subsections. diff --git a/Documentation/config/log.txt b/Documentation/config/log.txt index 456eb07800..5250ba45fb 100644 --- a/Documentation/config/log.txt +++ b/Documentation/config/log.txt @@ -18,6 +18,11 @@ log.decorate:: names are shown. This is the same as the `--decorate` option of the `git log`. +log.initialDecorationSet:: + By default, `git log` only shows decorations for certain known ref + namespaces. If 'all' is specified, then show all refs as + decorations. + log.excludeDecoration:: Exclude the specified patterns from the log decorations. This is similar to the `--decorate-refs-exclude` command-line option, but diff --git a/Documentation/config/lsrefs.txt b/Documentation/config/lsrefs.txt index adeda0f24d..3d88fb0bad 100644 --- a/Documentation/config/lsrefs.txt +++ b/Documentation/config/lsrefs.txt @@ -1,7 +1,7 @@ lsrefs.unborn:: May be "advertise" (the default), "allow", or "ignore". If "advertise", the server will respond to the client sending "unborn" (as described in - protocol-v2.txt) and will advertise support for this feature during the + linkgit:gitprotocol-v2[5]) and will advertise support for this feature during the protocol v2 capability advertisement. "allow" is the same as "advertise" except that the server will not advertise support for this feature; this is useful for load-balanced servers that cannot be diff --git a/Documentation/config/pack.txt b/Documentation/config/pack.txt index ad7f73a1ea..3e581eab84 100644 --- a/Documentation/config/pack.txt +++ b/Documentation/config/pack.txt @@ -166,7 +166,7 @@ permuted into their appropriate location when writing a new bitmap. pack.writeReverseIndex:: When true, git will write a corresponding .rev file (see: - link:../technical/pack-format.html[Documentation/technical/pack-format.txt]) + linkgit:gitformat-pack[5]) for each new packfile that it writes in all places except for linkgit:git-fast-import[1] and in the bulk checkin mechanism. Defaults to false. diff --git a/Documentation/config/protocol.txt b/Documentation/config/protocol.txt index 756591d77b..5760381851 100644 --- a/Documentation/config/protocol.txt +++ b/Documentation/config/protocol.txt @@ -58,6 +58,6 @@ protocol.version:: * `1` - the original wire protocol with the addition of a version string in the initial response from the server. -* `2` - link:technical/protocol-v2.html[wire protocol version 2]. +* `2` - Wire protocol version 2, see linkgit:gitprotocol-v2[5]. -- diff --git a/Documentation/config/rebase.txt b/Documentation/config/rebase.txt index 8c979cb20f..f19bd0e040 100644 --- a/Documentation/config/rebase.txt +++ b/Documentation/config/rebase.txt @@ -21,6 +21,9 @@ rebase.autoStash:: `--autostash` options of linkgit:git-rebase[1]. Defaults to false. +rebase.updateRefs:: + If set to true enable `--update-refs` option by default. + rebase.missingCommitsCheck:: If set to "warn", git rebase -i will print a warning if some commits are removed (e.g. a line was deleted), however the diff --git a/Documentation/git-add.txt b/Documentation/git-add.txt index 11eb70f16c..9b37f35654 100644 --- a/Documentation/git-add.txt +++ b/Documentation/git-add.txt @@ -188,7 +188,9 @@ for "git add --no-all <pathspec>...", i.e. ignored removed files. forcibly add them again to the index. This is useful after changing `core.autocrlf` configuration or the `text` attribute in order to correct files added with wrong CRLF/LF line endings. - This option implies `-u`. + This option implies `-u`. Lone CR characters are untouched, thus + while a CRLF cleans to LF, a CRCRLF sequence is only partially + cleaned to CRLF. --chmod=(+|-)x:: Override the executable bit of the added files. The executable diff --git a/Documentation/git-am.txt b/Documentation/git-am.txt index 09107fb106..320da6c4f7 100644 --- a/Documentation/git-am.txt +++ b/Documentation/git-am.txt @@ -112,10 +112,7 @@ default. You can use `--no-utf8` to override this. am.threeWay configuration variable. For more information, see am.threeWay in linkgit:git-config[1]. ---rerere-autoupdate:: ---no-rerere-autoupdate:: - Allow the rerere mechanism to update the index with the - result of auto-conflict resolution if possible. +include::rerere-options.txt[] --ignore-space-change:: --ignore-whitespace:: diff --git a/Documentation/git-bugreport.txt b/Documentation/git-bugreport.txt index d8817bf3ce..eca726e579 100644 --- a/Documentation/git-bugreport.txt +++ b/Documentation/git-bugreport.txt @@ -9,6 +9,7 @@ SYNOPSIS -------- [verse] 'git bugreport' [(-o | --output-directory) <path>] [(-s | --suffix) <format>] + [--diagnose[=<mode>]] DESCRIPTION ----------- @@ -31,6 +32,10 @@ The following information is captured automatically: - A list of enabled hooks - $SHELL +Additional information may be gathered into a separate zip archive using the +`--diagnose` option, and can be attached alongside the bugreport document to +provide additional context to readers. + This tool is invoked via the typical Git setup process, which means that in some cases, it might not be able to launch - for example, if a relevant config file is unreadable. In this kind of scenario, it may be helpful to manually gather @@ -49,6 +54,19 @@ OPTIONS named 'git-bugreport-<formatted suffix>'. This should take the form of a strftime(3) format string; the current local time will be used. +--no-diagnose:: +--diagnose[=<mode>]:: + Create a zip archive of supplemental information about the user's + machine, Git client, and repository state. The archive is written to the + same output directory as the bug report and is named + 'git-diagnostics-<formatted suffix>'. ++ +Without `mode` specified, the diagnostic archive will contain the default set of +statistics reported by `git diagnose`. An optional `mode` value may be specified +to change which information is included in the archive. See +linkgit:git-diagnose[1] for the list of valid values for `mode` and details +about their usage. + GIT --- Part of the linkgit:git[1] suite diff --git a/Documentation/git-bundle.txt b/Documentation/git-bundle.txt index 7685b57045..6da6172243 100644 --- a/Documentation/git-bundle.txt +++ b/Documentation/git-bundle.txt @@ -56,10 +56,8 @@ using "thin packs", bundles created using exclusions are smaller in size. That they're "thin" under the hood is merely noted here as a curiosity, and as a reference to other documentation. -See link:technical/bundle-format.html[the `bundle-format` -documentation] for more details and the discussion of "thin pack" in -link:technical/pack-format.html[the pack format documentation] for -further details. +See linkgit:gitformat-bundle[5] for more details and the discussion of +"thin pack" in linkgit:gitformat-pack[5] for further details. OPTIONS ------- @@ -77,7 +75,7 @@ verify <file>:: commits exist and are fully linked in the current repository. Then, 'git bundle' prints a list of missing commits, if any. Finally, information about additional capabilities, such as "object - filter", is printed. See "Capabilities" in link:technical/bundle-format.html + filter", is printed. See "Capabilities" in linkgit:gitformat-bundle[5] for more information. The exit code is zero for success, but will be nonzero if the bundle file is invalid. @@ -337,6 +335,11 @@ You can also see what references it offers: $ git ls-remote mybundle ---------------- +FILE FORMAT +----------- + +See linkgit:gitformat-bundle[5]. + GIT --- Part of the linkgit:git[1] suite diff --git a/Documentation/git-cat-file.txt b/Documentation/git-cat-file.txt index 24a811f0ef..ec30b5c574 100644 --- a/Documentation/git-cat-file.txt +++ b/Documentation/git-cat-file.txt @@ -14,7 +14,7 @@ SYNOPSIS 'git cat-file' (-t | -s) [--allow-unknown-type] <object> 'git cat-file' (--batch | --batch-check | --batch-command) [--batch-all-objects] [--buffer] [--follow-symlinks] [--unordered] - [--textconv | --filters] + [--textconv | --filters] [-z] 'git cat-file' (--textconv | --filters) [<rev>:<path|tree-ish> | --path=<path|tree-ish> <rev>] @@ -63,6 +63,12 @@ OPTIONS or to ask for a "blob" with `<object>` being a tag object that points at it. +--[no-]mailmap:: +--[no-]use-mailmap:: + Use mailmap file to map author, committer and tagger names + and email addresses to canonical real names and email addresses. + See linkgit:git-shortlog[1]. + --textconv:: Show the content as transformed by a textconv filter. In this case, `<object>` has to be of the form `<tree-ish>:<path>`, or `:<path>` in @@ -207,6 +213,11 @@ respectively print: /etc/passwd -- +-z:: + Only meaningful with `--batch`, `--batch-check`, or + `--batch-command`; input is NUL-delimited instead of + newline-delimited. + OUTPUT ------ diff --git a/Documentation/git-cherry-pick.txt b/Documentation/git-cherry-pick.txt index 78dcc9171f..1e8ac9df60 100644 --- a/Documentation/git-cherry-pick.txt +++ b/Documentation/git-cherry-pick.txt @@ -156,10 +156,7 @@ effect to your index in a row. Pass the merge strategy-specific option through to the merge strategy. See linkgit:git-merge[1] for details. ---rerere-autoupdate:: ---no-rerere-autoupdate:: - Allow the rerere mechanism to update the index with the - result of auto-conflict resolution if possible. +include::rerere-options.txt[] SEQUENCER SUBCOMMANDS --------------------- diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index 632bd1348e..d032d971dd 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -323,6 +323,13 @@ or `--mirror` is given) for `host.xz:foo/.git`). Cloning into an existing directory is only allowed if the directory is empty. +--bundle-uri=<uri>:: + Before fetching from the remote, fetch a bundle from the given + `<uri>` and unbundle the data into the local repository. The refs + in the bundle will be stored under the hidden `refs/bundle/*` + namespace. This option is incompatible with `--depth`, + `--shallow-since`, and `--shallow-exclude`. + :git-clone: 1 include::urls.txt[] diff --git a/Documentation/git-commit-graph.txt b/Documentation/git-commit-graph.txt index e1f48c95b3..047decdb65 100644 --- a/Documentation/git-commit-graph.txt +++ b/Documentation/git-commit-graph.txt @@ -143,6 +143,11 @@ $ git rev-parse HEAD | git commit-graph write --stdin-commits --append ------------------------------------------------ +FILE FORMAT +----------- + +see linkgit:gitformat-commit-graph[5]. + GIT --- Part of the linkgit:git[1] suite diff --git a/Documentation/git-cvsserver.txt b/Documentation/git-cvsserver.txt index 4dc57ed254..53f111bc0a 100644 --- a/Documentation/git-cvsserver.txt +++ b/Documentation/git-cvsserver.txt @@ -63,11 +63,10 @@ Print version information and exit Print usage information and exit <directory>:: -You can specify a list of allowed directories. If no directories -are given, all are allowed. This is an additional restriction, gitcvs -access still needs to be enabled by the `gitcvs.enabled` config option -unless `--export-all` was given, too. - +The remaining arguments provide a list of directories. If no directories +are given, then all are allowed. Repositories within these directories +still require the `gitcvs.enabled` config option, unless `--export-all` +is specified. LIMITATIONS ----------- @@ -311,11 +310,13 @@ ENVIRONMENT These variables obviate the need for command-line options in some circumstances, allowing easier restricted usage through git-shell. -GIT_CVSSERVER_BASE_PATH takes the place of the argument to --base-path. +GIT_CVSSERVER_BASE_PATH:: + This variable replaces the argument to --base-path. -GIT_CVSSERVER_ROOT specifies a single-directory whitelist. The -repository must still be configured to allow access through -git-cvsserver, as described above. +GIT_CVSSERVER_ROOT:: + This variable specifies a single directory, replacing the + `<directory>...` argument list. The repository still requires the + `gitcvs.enabled` config option, unless `--export-all` is specified. When these environment variables are set, the corresponding command-line arguments may not be used. diff --git a/Documentation/git-daemon.txt b/Documentation/git-daemon.txt index fdc28c041c..236df516c7 100644 --- a/Documentation/git-daemon.txt +++ b/Documentation/git-daemon.txt @@ -32,8 +32,8 @@ that service if it is enabled. It verifies that the directory has the magic file "git-daemon-export-ok", and it will refuse to export any Git directory that hasn't explicitly been marked for export this way (unless the `--export-all` parameter is specified). If you -pass some directory paths as 'git daemon' arguments, you can further restrict -the offers to a whitelist comprising of those. +pass some directory paths as 'git daemon' arguments, the offers are limited to +repositories within those directories. By default, only `upload-pack` service is enabled, which serves 'git fetch-pack' and 'git ls-remote' clients, which are invoked @@ -50,7 +50,7 @@ OPTIONS Match paths exactly (i.e. don't allow "/foo/repo" when the real path is "/foo/repo.git" or "/foo/repo/.git") and don't do user-relative paths. 'git daemon' will refuse to start when this option is enabled and no - whitelist is specified. + directory arguments are provided. --base-path=<path>:: Remap all the path requests as relative to the given path. @@ -73,7 +73,7 @@ OPTIONS %IP for the server's IP address, %P for the port number, and %D for the absolute path of the named repository. After interpolation, the path is validated against the directory - whitelist. + list. --export-all:: Allow pulling from all directories that look like Git repositories @@ -218,9 +218,11 @@ standard output to be sent to the requestor as an error message when it declines the service. <directory>:: - A directory to add to the whitelist of allowed directories. Unless - --strict-paths is specified this will also include subdirectories - of each named directory. + The remaining arguments provide a list of directories. If any + directories are specified, then the `git-daemon` process will + serve a requested directory only if it is contained in one of + these directories. If `--strict-paths` is specified, then the + requested directory must match one of these directories exactly. SERVICES -------- @@ -264,9 +266,8 @@ git 9418/tcp # Git Version Control System 'git daemon' as inetd server:: To set up 'git daemon' as an inetd service that handles any - repository under the whitelisted set of directories, /pub/foo - and /pub/bar, place an entry like the following into - /etc/inetd all on one line: + repository within `/pub/foo` or `/pub/bar`, place an entry like + the following into `/etc/inetd` all on one line: + ------------------------------------------------ git stream tcp nowait nobody /usr/bin/git diff --git a/Documentation/git-diagnose.txt b/Documentation/git-diagnose.txt new file mode 100644 index 0000000000..3ec8cc7ad7 --- /dev/null +++ b/Documentation/git-diagnose.txt @@ -0,0 +1,65 @@ +git-diagnose(1) +================ + +NAME +---- +git-diagnose - Generate a zip archive of diagnostic information + +SYNOPSIS +-------- +[verse] +'git diagnose' [(-o | --output-directory) <path>] [(-s | --suffix) <format>] + [--mode=<mode>] + +DESCRIPTION +----------- +Collects detailed information about the user's machine, Git client, and +repository state and packages that information into a zip archive. The +generated archive can then, for example, be shared with the Git mailing list to +help debug an issue or serve as a reference for independent debugging. + +By default, the following information is captured in the archive: + + * 'git version --build-options' + * The path to the repository root + * The available disk space on the filesystem + * The name and size of each packfile, including those in alternate object + stores + * The total count of loose objects, as well as counts broken down by + `.git/objects` subdirectory + +Additional information can be collected by selecting a different diagnostic mode +using the `--mode` option. + +This tool differs from linkgit:git-bugreport[1] in that it collects much more +detailed information with a greater focus on reporting the size and data shape +of repository contents. + +OPTIONS +------- +-o <path>:: +--output-directory <path>:: + Place the resulting diagnostics archive in `<path>` instead of the + current directory. + +-s <format>:: +--suffix <format>:: + Specify an alternate suffix for the diagnostics archive name, to create + a file named 'git-diagnostics-<formatted suffix>'. This should take the + form of a strftime(3) format string; the current local time will be + used. + +--mode=(stats|all):: + Specify the type of diagnostics that should be collected. The default behavior + of 'git diagnose' is equivalent to `--mode=stats`. ++ +The `--mode=all` option collects everything included in `--mode=stats`, as well +as copies of `.git`, `.git/hooks`, `.git/info`, `.git/logs`, and +`.git/objects/info` directories. This additional information may be sensitive, +as it can be used to reconstruct the full contents of the diagnosed repository. +Users should exercise caution when sharing an archive generated with +`--mode=all`. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-help.txt b/Documentation/git-help.txt index 239c68db45..2b0b5e390d 100644 --- a/Documentation/git-help.txt +++ b/Documentation/git-help.txt @@ -9,14 +9,16 @@ SYNOPSIS -------- [verse] 'git help' [-a|--all] [--[no-]verbose] [--[no-]external-commands] [--[no-]aliases] -'git help' [[-i|--info] [-m|--man] [-w|--web]] [<command>|<guide>] +'git help' [[-i|--info] [-m|--man] [-w|--web]] [<command>|<doc>] 'git help' [-g|--guides] 'git help' [-c|--config] +'git help' [--user-interfaces] +'git help' [--developer-interfaces] DESCRIPTION ----------- -With no options and no '<command>' or '<guide>' given, the synopsis of the 'git' +With no options and no '<command>' or '<doc>' given, the synopsis of the 'git' command and a list of the most commonly used Git commands are printed on the standard output. @@ -26,8 +28,8 @@ printed on the standard output. If the option `--guides` or `-g` is given, a list of the Git concept guides is also printed on the standard output. -If a command, or a guide, is given, a manual page for that command or -guide is brought up. The 'man' program is used by default for this +If a command or other documentation is given, the relevant manual page +will be brought up. The 'man' program is used by default for this purpose, but this can be overridden by other options or configuration variables. @@ -69,6 +71,23 @@ OPTIONS --guides:: Prints a list of the Git concept guides on the standard output. +--user-interfaces:: + Prints a list of the repository, command and file interfaces + documentation on the standard output. ++ +In-repository file interfaces such as `.git/info/exclude` are +documented here (see linkgit:gitrepository-layout[5]), as well as +in-tree configuration such as `.mailmap` (see linkgit:gitmailmap[5]). ++ +This section of the documentation also covers general or widespread +user-interface conventions (e.g. linkgit:gitcli[7]), and +pseudo-configuration such as the file-based `.git/hooks/*` interface +described in linkgit:githooks[5]. + +--developer-interfaces:: + Print list of file formats, protocols and other developer + interfaces documentation on the standard output. + -i:: --info:: Display manual page for the command in the 'info' format. The diff --git a/Documentation/git-log.txt b/Documentation/git-log.txt index 20e87cecf4..b1285aee3c 100644 --- a/Documentation/git-log.txt +++ b/Documentation/git-log.txt @@ -45,13 +45,23 @@ OPTIONS --decorate-refs=<pattern>:: --decorate-refs-exclude=<pattern>:: - If no `--decorate-refs` is given, pretend as if all refs were - included. For each candidate, do not use it for decoration if it + For each candidate reference, do not use it for decoration if it matches any patterns given to `--decorate-refs-exclude` or if it doesn't match any of the patterns given to `--decorate-refs`. The `log.excludeDecoration` config option allows excluding refs from the decorations, but an explicit `--decorate-refs` pattern will override a match in `log.excludeDecoration`. ++ +If none of these options or config settings are given, then references are +used as decoration if they match `HEAD`, `refs/heads/`, `refs/remotes/`, +`refs/stash/`, or `refs/tags/`. + +--clear-decorations:: + When specified, this option clears all previous `--decorate-refs` + or `--decorate-refs-exclude` options and relaxes the default + decoration filter to include all references. This option is + assumed if the config value `log.initialDecorationSet` is set to + `all`. --source:: Print out the ref name given on the command line by which each diff --git a/Documentation/git-ls-files.txt b/Documentation/git-ls-files.txt index 0dabf3f0dd..d7986419c2 100644 --- a/Documentation/git-ls-files.txt +++ b/Documentation/git-ls-files.txt @@ -20,7 +20,7 @@ SYNOPSIS [--exclude-standard] [--error-unmatch] [--with-tree=<tree-ish>] [--full-name] [--recurse-submodules] - [--abbrev[=<n>]] [--] [<file>...] + [--abbrev[=<n>]] [--format=<format>] [--] [<file>...] DESCRIPTION ----------- @@ -192,6 +192,13 @@ followed by the ("attr/<eolattr>"). to the contained files. Sparse directories will be shown with a trailing slash, such as "x/" for a sparse directory "x". +--format=<format>:: + A string that interpolates `%(fieldname)` from the result being shown. + It also interpolates `%%` to `%`, and `%xx` where `xx` are hex digits + interpolates to character with hex code `xx`; for example `%00` + interpolates to `\0` (NUL), `%09` to `\t` (TAB) and %0a to `\n` (LF). + --format cannot be combined with `-s`, `-o`, `-k`, `-t`, `--resolve-undo` + and `--eol`. \--:: Do not interpret any more arguments as options. @@ -223,6 +230,36 @@ quoted as explained for the configuration variable `core.quotePath` (see linkgit:git-config[1]). Using `-z` the filename is output verbatim and the line is terminated by a NUL byte. +It is possible to print in a custom format by using the `--format` +option, which is able to interpolate different fields using +a `%(fieldname)` notation. For example, if you only care about the +"objectname" and "path" fields, you can execute with a specific +"--format" like + + git ls-files --format='%(objectname) %(path)' + +FIELD NAMES +----------- +The way each path is shown can be customized by using the +`--format=<format>` option, where the %(fieldname) in the +<format> string for various aspects of the index entry are +interpolated. The following "fieldname" are understood: + +objectmode:: + The mode of the file which is recorded in the index. +objectname:: + The name of the file which is recorded in the index. +stage:: + The stage of the file which is recorded in the index. +eolinfo:index:: +eolinfo:worktree:: + The <eolinfo> (see the description of the `--eol` option) of + the contents in the index or in the worktree for the path. +eolattr:: + The <eolattr> (see the description of the `--eol` option) + that applies to the path. +path:: + The pathname of the file which is recorded in the index. EXCLUDE PATTERNS ---------------- diff --git a/Documentation/git-merge.txt b/Documentation/git-merge.txt index 3125473cc1..fee1dc2df2 100644 --- a/Documentation/git-merge.txt +++ b/Documentation/git-merge.txt @@ -90,10 +90,7 @@ invocations. The automated message can include the branch description. If `--log` is specified, a shortlog of the commits being merged will be appended to the specified message. ---rerere-autoupdate:: ---no-rerere-autoupdate:: - Allow the rerere mechanism to update the index with the - result of auto-conflict resolution if possible. +include::rerere-options.txt[] --overwrite-ignore:: --no-overwrite-ignore:: diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index c588fb91af..a48c3d5ea6 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -128,8 +128,8 @@ $ git multi-pack-index verify SEE ALSO -------- See link:technical/multi-pack-index.html[The Multi-Pack-Index Design -Document] and link:technical/pack-format.html[The Multi-Pack-Index -Format] for more information on the multi-pack-index feature. +Document] and linkgit:gitformat-pack[5] for more information on the +multi-pack-index feature and its file format. GIT diff --git a/Documentation/git-rebase.txt b/Documentation/git-rebase.txt index a872ab0fbd..1877942180 100644 --- a/Documentation/git-rebase.txt +++ b/Documentation/git-rebase.txt @@ -376,10 +376,7 @@ See also INCOMPATIBLE OPTIONS below. + See also INCOMPATIBLE OPTIONS below. ---rerere-autoupdate:: ---no-rerere-autoupdate:: - Allow the rerere mechanism to update the index with the - result of auto-conflict resolution if possible. +include::rerere-options.txt[] -S[<keyid>]:: --gpg-sign[=<keyid>]:: @@ -612,6 +609,15 @@ provided. Otherwise an explicit `--no-reschedule-failed-exec` at the start would be overridden by the presence of `rebase.rescheduleFailedExec=true` configuration. +--update-refs:: +--no-update-refs:: + Automatically force-update any branches that point to commits that + are being rebased. Any branches that are checked out in a worktree + are not updated in this way. ++ +If the configuration variable `rebase.updateRefs` is set, then this option +can be used to override and disable this setting. + INCOMPATIBLE OPTIONS -------------------- @@ -635,6 +641,7 @@ are incompatible with the following options: * --empty= * --reapply-cherry-picks * --edit-todo + * --update-refs * --root when used in combination with --onto In addition, the following pairs of options are incompatible: diff --git a/Documentation/git-reflog.txt b/Documentation/git-reflog.txt index 5ced7ad4f8..db9d46edfa 100644 --- a/Documentation/git-reflog.txt +++ b/Documentation/git-reflog.txt @@ -22,7 +22,7 @@ depending on the subcommand: [--rewrite] [--updateref] [--stale-fix] [--dry-run | -n] [--verbose] [--all [--single-worktree] | <refs>...] 'git reflog delete' [--rewrite] [--updateref] - [--dry-run | -n] [--verbose] <ref>@\{<specifier>\}... + [--dry-run | -n] [--verbose] <ref>@{<specifier>}... 'git reflog exists' <ref> Reference logs, or "reflogs", record when the tips of branches and diff --git a/Documentation/git-revert.txt b/Documentation/git-revert.txt index 8463fe9cf7..0105a54c1a 100644 --- a/Documentation/git-revert.txt +++ b/Documentation/git-revert.txt @@ -112,10 +112,7 @@ effect to your index in a row. Pass the merge strategy-specific option through to the merge strategy. See linkgit:git-merge[1] for details. ---rerere-autoupdate:: ---no-rerere-autoupdate:: - Allow the rerere mechanism to update the index with the - result of auto-conflict resolution if possible. +include::rerere-options.txt[] --reference:: Instead of starting the body of the log message with "This diff --git a/Documentation/git-upload-pack.txt b/Documentation/git-upload-pack.txt index 8f87b23ea8..3f89d64077 100644 --- a/Documentation/git-upload-pack.txt +++ b/Documentation/git-upload-pack.txt @@ -39,10 +39,9 @@ OPTIONS --http-backend-info-refs:: Used by linkgit:git-http-backend[1] to serve up `$GIT_URL/info/refs?service=git-upload-pack` requests. See - "Smart Clients" in link:technical/http-protocol.html[the HTTP - transfer protocols] documentation and "HTTP Transport" in - link:technical/protocol-v2.html[the Git Wire Protocol, Version - 2] documentation. Also understood by + "Smart Clients" in linkgit:gitprotocol-http[5] and "HTTP + Transport" in in the linkgit:gitprotocol-v2[5] + documentation. Also understood by linkgit:git-receive-pack[1]. <directory>:: diff --git a/Documentation/git.txt b/Documentation/git.txt index 302607a496..0ef7f5e4ec 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -339,6 +339,23 @@ The following documentation pages are guides about Git concepts. include::cmds-guide.txt[] +Repository, command and file interfaces +--------------------------------------- + +This documentation discusses repository and command interfaces which +users are expected to interact with directly. See `--user-formats` in +linkgit:git-help[1] for more details on the critera. + +include::cmds-userinterfaces.txt[] + +File formats, protocols and other developer interfaces +------------------------------------------------------ + +This documentation discusses file formats, over-the-wire protocols and +other git developer interfaces. See `--developer-interfaces` in +linkgit:git-help[1]. + +include::cmds-developerinterfaces.txt[] Configuration Mechanism ----------------------- @@ -885,9 +902,7 @@ for full details. If set to a colon-separated list of protocols, behave as if `protocol.allow` is set to `never`, and each of the listed protocols has `protocol.<name>.allow` set to `always` - (overriding any existing configuration). In other words, any - protocol not mentioned will be disallowed (i.e., this is a - whitelist, not a blacklist). See the description of + (overriding any existing configuration). See the description of `protocol.allow` in linkgit:git-config[1] for more details. `GIT_PROTOCOL_FROM_USER`:: diff --git a/Documentation/technical/bundle-format.txt b/Documentation/gitformat-bundle.txt index b9be8644cf..00e0a20e65 100644 --- a/Documentation/technical/bundle-format.txt +++ b/Documentation/gitformat-bundle.txt @@ -1,11 +1,33 @@ -= Git bundle v2 format +gitformat-bundle(5) +=================== -The Git bundle format is a format that represents both refs and Git objects. +NAME +---- +gitformat-bundle - The bundle file format + + +SYNOPSIS +-------- +[verse] +*.bundle +*.bdl + +DESCRIPTION +----------- + +The Git bundle format is a format that represents both refs and Git +objects. A bundle is a header in a format similar to +linkgit:git-show-ref[1] followed by a pack in *.pack format. -== Format +The format is created and read by the linkgit:git-bundle[1] command, +and supported by e.g. linkgit:git-fetch[1] and linkgit:git-clone[1]. + + +FORMAT +------ We will use ABNF notation to define the Git bundle format. See -protocol-common.txt for the details. +linkgit:gitprotocol-common[5] for the details. A v2 bundle looks like this: @@ -36,7 +58,9 @@ value = *(%01-09 / %0b-FF) pack = ... ; packfile ---- -== Semantics + +SEMANTICS +--------- A Git bundle consists of several parts. @@ -62,13 +86,15 @@ In the bundle format, there can be a comment following a prerequisite obj-id. This is a comment and it has no specific meaning. The writer of the bundle MAY put any string here. The reader of the bundle MUST ignore the comment. -=== Note on the shallow clone and a Git bundle +Note on the shallow clone and a Git bundle +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Note that the prerequisites does not represent a shallow-clone boundary. The semantics of the prerequisites and the shallow-clone boundaries are different, and the Git bundle v2 format cannot represent a shallow clone repository. -== Capabilities +CAPABILITIES +------------ Because there is no opportunity for negotiation, unknown capabilities cause 'git bundle' to abort. @@ -79,3 +105,7 @@ bundle' to abort. * `filter` specifies an object filter as in the `--filter` option in linkgit:git-rev-list[1]. The resulting pack-file must be marked as a `.promisor` pack-file after it is unbundled. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/chunk-format.txt b/Documentation/gitformat-chunk.txt index 593614fced..57202ede27 100644 --- a/Documentation/technical/chunk-format.txt +++ b/Documentation/gitformat-chunk.txt @@ -1,12 +1,25 @@ -Chunk-based file formats -======================== +gitformat-chunk(5) +================== + +NAME +---- +gitformat-chunk - Chunk-based file formats + +SYNOPSIS +-------- + +Used by linkgit:gitformat-commit-graph[5] and the "MIDX" format (see +the pack format documentation in linkgit:gitformat-pack[5]). + +DESCRIPTION +----------- Some file formats in Git use a common concept of "chunks" to describe sections of the file. This allows structured access to a large file by scanning a small "table of contents" for the remaining data. This common format is used by the `commit-graph` and `multi-pack-index` files. See -link:technical/pack-format.html[the `multi-pack-index` format] and -link:technical/commit-graph-format.html[the `commit-graph` format] for +the `multi-pack-index` format in linkgit:gitformat-pack[5] and +the `commit-graph` format in linkgit:gitformat-commit-graph[5] for how they use the chunks to describe structured data. A chunk-based file format begins with some header information custom to @@ -108,9 +121,13 @@ for future formats: * *commit-graph:* see `write_commit_graph_file()` and `parse_commit_graph()` in `commit-graph.c` for how the chunk-format API is used to write and parse the commit-graph file format documented in - link:technical/commit-graph-format.html[the commit-graph file format]. + the commit-graph file format in linkgit:gitformat-commit-graph[5]. * *multi-pack-index:* see `write_midx_internal()` and `load_multi_pack_index()` in `midx.c` for how the chunk-format API is used to write and parse the multi-pack-index file format documented in - link:technical/pack-format.html[the multi-pack-index file format]. + the multi-pack-index file format section of linkgit:gitformat-pack[5]. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/commit-graph-format.txt b/Documentation/gitformat-commit-graph.txt index 484b185ba9..7324665716 100644 --- a/Documentation/technical/commit-graph-format.txt +++ b/Documentation/gitformat-commit-graph.txt @@ -1,5 +1,18 @@ -Git commit graph format -======================= +gitformat-commit-graph(5) +========================= + +NAME +---- +gitformat-commit-graph - Git commit graph format + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/info/commit-graph +$GIT_DIR/objects/info/commit-graphs/* + +DESCRIPTION +----------- The Git commit graph stores a list of commit OIDs and some associated metadata, including: @@ -30,7 +43,7 @@ and hash type. All multi-byte numbers are in network byte order. -HEADER: +=== HEADER: 4-byte signature: The signature is: {'C', 'G', 'P', 'H'} @@ -52,7 +65,7 @@ HEADER: We infer the length (H*B) of the Base Graphs chunk from this value. -CHUNK LOOKUP: +=== CHUNK LOOKUP: (C + 1) * 12 bytes listing the table of contents for the chunks: First 4 bytes describe the chunk id. Value 0 is a terminating label. @@ -62,23 +75,23 @@ CHUNK LOOKUP: ID appears at most once. The CHUNK LOOKUP matches the table of contents from - link:technical/chunk-format.html[the chunk-based file format]. + the chunk-based file format, see linkgit:gitformat-chunk[5] The remaining data in the body is described one chunk at a time, and these chunks may be given in any order. Chunks are required unless otherwise specified. -CHUNK DATA: +=== CHUNK DATA: - OID Fanout (ID: {'O', 'I', 'D', 'F'}) (256 * 4 bytes) +==== OID Fanout (ID: {'O', 'I', 'D', 'F'}) (256 * 4 bytes) The ith entry, F[i], stores the number of OIDs with first byte at most i. Thus F[255] stores the total number of commits (N). - OID Lookup (ID: {'O', 'I', 'D', 'L'}) (N * H bytes) +==== OID Lookup (ID: {'O', 'I', 'D', 'L'}) (N * H bytes) The OIDs for all commits in the graph, sorted in ascending order. - Commit Data (ID: {'C', 'D', 'A', 'T' }) (N * (H + 16) bytes) +==== Commit Data (ID: {'C', 'D', 'A', 'T' }) (N * (H + 16) bytes) * The first H bytes are for the OID of the root tree. * The next 8 bytes are for the positions of the first two parents of the ith commit. Stores value 0x70000000 if no parent in that @@ -93,7 +106,7 @@ CHUNK DATA: 2 bits of the lowest byte, storing the 33rd and 34th bit of the commit time. - Generation Data (ID: {'G', 'D', 'A', '2' }) (N * 4 bytes) [Optional] +==== Generation Data (ID: {'G', 'D', 'A', '2' }) (N * 4 bytes) [Optional] * This list of 4-byte values store corrected commit date offsets for the commits, arranged in the same order as commit data chunk. * If the corrected commit date offset cannot be stored within 31 bits, @@ -104,7 +117,7 @@ CHUNK DATA: by compatible versions of Git and in case of split commit-graph chains, the topmost layer also has Generation Data chunk. - Generation Data Overflow (ID: {'G', 'D', 'O', '2' }) [Optional] +==== Generation Data Overflow (ID: {'G', 'D', 'O', '2' }) [Optional] * This list of 8-byte values stores the corrected commit date offsets for commits with corrected commit date offsets that cannot be stored within 31 bits. @@ -112,7 +125,7 @@ CHUNK DATA: chunk is present and atleast one corrected commit date offset cannot be stored within 31 bits. - Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional] +==== Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional] This list of 4-byte values store the second through nth parents for all octopus merges. The second parent value in the commit data stores an array position within this list along with the most-significant bit @@ -120,14 +133,14 @@ CHUNK DATA: positions for the parents until reaching a value with the most-significant bit on. The other bits correspond to the position of the last parent. - Bloom Filter Index (ID: {'B', 'I', 'D', 'X'}) (N * 4 bytes) [Optional] +==== Bloom Filter Index (ID: {'B', 'I', 'D', 'X'}) (N * 4 bytes) [Optional] * The ith entry, BIDX[i], stores the number of bytes in all Bloom filters from commit 0 to commit i (inclusive) in lexicographic order. The Bloom filter for the i-th commit spans from BIDX[i-1] to BIDX[i] (plus header length), where BIDX[-1] is 0. * The BIDX chunk is ignored if the BDAT chunk is not present. - Bloom Filter Data (ID: {'B', 'D', 'A', 'T'}) [Optional] +==== Bloom Filter Data (ID: {'B', 'D', 'A', 'T'}) [Optional] * It starts with header consisting of three unsigned 32-bit integers: - Version of the hash algorithm being used. We currently only support value 1 which corresponds to the 32-bit version of the murmur3 hash @@ -147,13 +160,13 @@ CHUNK DATA: of length one, with either all bits set to zero or one respectively. * The BDAT chunk is present if and only if BIDX is present. - Base Graphs List (ID: {'B', 'A', 'S', 'E'}) [Optional] +==== Base Graphs List (ID: {'B', 'A', 'S', 'E'}) [Optional] This list of H-byte hashes describe a set of B commit-graph files that form a commit-graph chain. The graph position for the ith commit in this file's OID Lookup chunk is equal to i plus the number of commits in all base graphs. If B is non-zero, this chunk must exist. -TRAILER: +=== TRAILER: H-byte HASH-checksum of all of the above. @@ -164,3 +177,7 @@ the number '2' in their chunk IDs because a previous version of Git wrote possibly erroneous data in these chunks with the IDs "GDAT" and "GDOV". By changing the IDs, newer versions of Git will silently ignore those older chunks and write the new information without trusting the incorrect data. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/index-format.txt b/Documentation/gitformat-index.txt index 65da0daaa5..015cb21bdc 100644 --- a/Documentation/technical/index-format.txt +++ b/Documentation/gitformat-index.txt @@ -1,5 +1,19 @@ +gitformat-index(5) +================== + +NAME +---- +gitformat-index - Git index format + +SYNOPSIS +-------- +[verse] +$GIT_DIR/index + +DESCRIPTION +----------- + Git index format -================ == The Git index file has the following format @@ -26,8 +40,6 @@ Git index format Extensions are identified by signature. Optional extensions can be ignored if Git does not understand them. - Git currently supports cache tree and resolve undo extensions. - 4-byte extension signature. If the first byte is 'A'..'Z' the extension is optional and can be ignored. @@ -127,7 +139,7 @@ Git index format entry is encoded as if the path name for the previous entry is an empty string). At the beginning of an entry, an integer N in the variable width encoding (the same encoding as the offset is encoded - for OFS_DELTA pack entries; see pack-format.txt) is stored, followed + for OFS_DELTA pack entries; see linkgit:gitformat-pack[5]) is stored, followed by a NUL-terminated string S. Removing N bytes from the end of the path name for the previous entry, and replacing it with the string S yields the path name for this entry. @@ -404,3 +416,7 @@ The remaining data of each directory block is grouped by type: with signature { 's', 'd', 'i', 'r' }. Like the split-index extension, tools should avoid interacting with a sparse index unless they understand this extension. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/pack-format.txt b/Documentation/gitformat-pack.txt index b520aa9c45..e06af02f21 100644 --- a/Documentation/technical/pack-format.txt +++ b/Documentation/gitformat-pack.txt @@ -1,5 +1,30 @@ -Git pack format -=============== +gitformat-pack(5) +================= + +NAME +---- +gitformat-pack - Git pack format + + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/pack/pack-*.{pack,idx} +$GIT_DIR/objects/pack/pack-*.rev +$GIT_DIR/objects/pack/pack-*.mtimes +$GIT_DIR/objects/pack/multi-pack-index + +DESCRIPTION +----------- + +The Git pack format is now Git stores most of its primary repository +data. Over the lietime af a repository loose objects (if any) and +smaller packs are consolidated into larger pack(s). See +linkgit:git-gc[1] and linkgit:git-pack-objects[1]. + +The pack format is also used over-the-wire, see +e.g. linkgit:gitprotocol-v2[5], as well as being a part of +other container formats in the case of linkgit:gitformat-bundle[5]. == Checksums and object IDs @@ -356,7 +381,7 @@ CHUNK LOOKUP: using the next chunk position if necessary.) The CHUNK LOOKUP matches the table of contents from - link:technical/chunk-format.html[the chunk-based file format]. + the chunk-based file format, see linkgit:gitformat-chunk[5]. The remaining data in the body is described one chunk at a time, and these chunks may be given in any order. Chunks are required unless @@ -482,3 +507,132 @@ packs arranged in MIDX order (with the preferred pack coming first). The MIDX's reverse index is stored in the optional 'RIDX' chunk within the MIDX itself. + +== cruft packs + +The cruft packs feature offer an alternative to Git's traditional mechanism of +removing unreachable objects. This document provides an overview of Git's +pruning mechanism, and how a cruft pack can be used instead to accomplish the +same. + +=== Background + +To remove unreachable objects from your repository, Git offers `git repack -Ad` +(see linkgit:git-repack[1]). Quoting from the documentation: + +---- +[...] unreachable objects in a previous pack become loose, unpacked objects, +instead of being left in the old pack. [...] loose unreachable objects will be +pruned according to normal expiry rules with the next 'git gc' invocation. +---- + +Unreachable objects aren't removed immediately, since doing so could race with +an incoming push which may reference an object which is about to be deleted. +Instead, those unreachable objects are stored as loose objects and stay that way +until they are older than the expiration window, at which point they are removed +by linkgit:git-prune[1]. + +Git must store these unreachable objects loose in order to keep track of their +per-object mtimes. If these unreachable objects were written into one big pack, +then either freshening that pack (because an object contained within it was +re-written) or creating a new pack of unreachable objects would cause the pack's +mtime to get updated, and the objects within it would never leave the expiration +window. Instead, objects are stored loose in order to keep track of the +individual object mtimes and avoid a situation where all cruft objects are +freshened at once. + +This can lead to undesirable situations when a repository contains many +unreachable objects which have not yet left the grace period. Having large +directories in the shards of `.git/objects` can lead to decreased performance in +the repository. But given enough unreachable objects, this can lead to inode +starvation and degrade the performance of the whole system. Since we +can never pack those objects, these repositories often take up a large amount of +disk space, since we can only zlib compress them, but not store them in delta +chains. + +=== Cruft packs + +A cruft pack eliminates the need for storing unreachable objects in a loose +state by including the per-object mtimes in a separate file alongside a single +pack containing all loose objects. + +A cruft pack is written by `git repack --cruft` when generating a new pack. +linkgit:git-pack-objects[1]'s `--cruft` option. Note that `git repack --cruft` +is a classic all-into-one repack, meaning that everything in the resulting pack is +reachable, and everything else is unreachable. Once written, the `--cruft` +option instructs `git repack` to generate another pack containing only objects +not packed in the previous step (which equates to packing all unreachable +objects together). This progresses as follows: + + 1. Enumerate every object, marking any object which is (a) not contained in a + kept-pack, and (b) whose mtime is within the grace period as a traversal + tip. + + 2. Perform a reachability traversal based on the tips gathered in the previous + step, adding every object along the way to the pack. + + 3. Write the pack out, along with a `.mtimes` file that records the per-object + timestamps. + +This mode is invoked internally by linkgit:git-repack[1] when instructed to +write a cruft pack. Crucially, the set of in-core kept packs is exactly the set +of packs which will not be deleted by the repack; in other words, they contain +all of the repository's reachable objects. + +When a repository already has a cruft pack, `git repack --cruft` typically only +adds objects to it. An exception to this is when `git repack` is given the +`--cruft-expiration` option, which allows the generated cruft pack to omit +expired objects instead of waiting for linkgit:git-gc[1] to expire those objects +later on. + +It is linkgit:git-gc[1] that is typically responsible for removing expired +unreachable objects. + +=== Caution for mixed-version environments + +Repositories that have cruft packs in them will continue to work with any older +version of Git. Note, however, that previous versions of Git which do not +understand the `.mtimes` file will use the cruft pack's mtime as the mtime for +all of the objects in it. In other words, do not expect older (pre-cruft pack) +versions of Git to interpret or even read the contents of the `.mtimes` file. + +Note that having mixed versions of Git GC-ing the same repository can lead to +unreachable objects never being completely pruned. This can happen under the +following circumstances: + + - An older version of Git running GC explodes the contents of an existing + cruft pack loose, using the cruft pack's mtime. + - A newer version running GC collects those loose objects into a cruft pack, + where the .mtime file reflects the loose object's actual mtimes, but the + cruft pack mtime is "now". + +Repeating this process will lead to unreachable objects not getting pruned as a +result of repeatedly resetting the objects' mtimes to the present time. + +If you are GC-ing repositories in a mixed version environment, consider omitting +the `--cruft` option when using linkgit:git-repack[1] and linkgit:git-gc[1], and +leaving the `gc.cruftPacks` configuration unset until all writers understand +cruft packs. + +=== Alternatives + +Notable alternatives to this design include: + + - The location of the per-object mtime data, and + - Storing unreachable objects in multiple cruft packs. + +On the location of mtime data, a new auxiliary file tied to the pack was chosen +to avoid complicating the `.idx` format. If the `.idx` format were ever to gain +support for optional chunks of data, it may make sense to consolidate the +`.mtimes` format into the `.idx` itself. + +Storing unreachable objects among multiple cruft packs (e.g., creating a new +cruft pack during each repacking operation including only unreachable objects +which aren't already stored in an earlier cruft pack) is significantly more +complicated to construct, and so aren't pursued here. The obvious drawback to +the current implementation is that the entire cruft pack must be re-written from +scratch. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/signature-format.txt b/Documentation/gitformat-signature.txt index 166721be6f..a249869faf 100644 --- a/Documentation/technical/signature-format.txt +++ b/Documentation/gitformat-signature.txt @@ -1,7 +1,18 @@ -Git signature format -==================== +gitformat-signature(5) +====================== -== Overview +NAME +---- +gitformat-signature - Git cryptographic signature formats + +SYNOPSIS +-------- +[verse] +<[tag|commit] object header(s)> +<over-the-wire protocol> + +DESCRIPTION +----------- Git uses cryptographic signatures in various places, currently objects (tags, commits, mergetags) and transactions (pushes). In every case, the command which @@ -200,3 +211,7 @@ Date: Wed Jun 15 09:13:29 2016 +0000 # gpg: There is no indication that the signature belongs to the owner. # Primary key fingerprint: D4BE 2231 1AD3 131E 5EDA 29A4 6109 2E85 B722 7189 ---- + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/protocol-capabilities.txt b/Documentation/gitprotocol-capabilities.txt index 9dfade930d..c6dcc7d565 100644 --- a/Documentation/technical/protocol-capabilities.txt +++ b/Documentation/gitprotocol-capabilities.txt @@ -1,8 +1,20 @@ -Git Protocol Capabilities -========================= +gitprotocol-capabilities(5) +=========================== + +NAME +---- +gitprotocol-capabilities - Protocol v0 and v1 capabilities + +SYNOPSIS +-------- +[verse] +<over-the-wire-protocol> + +DESCRIPTION +----------- NOTE: this document describes capabilities for versions 0 and 1 of the pack -protocol. For version 2, please refer to the link:protocol-v2.html[protocol-v2] +protocol. For version 2, please refer to the linkgit:gitprotocol-v2[5] doc. Servers SHOULD support all capabilities defined in this document. @@ -77,7 +89,7 @@ interleaved with S-R-Q. multi_ack_detailed ------------------ This is an extension of multi_ack that permits client to better -understand the server's in-memory state. See pack-protocol.txt, +understand the server's in-memory state. See linkgit:gitprotocol-pack[5], section "Packfile Negotiation" for more information. no-done @@ -281,7 +293,7 @@ a packfile upload and reference update. If the pushing client requests this capability, after unpacking and updating references the server will respond with whether the packfile unpacked successfully and if each reference was updated successfully. If any of those were not -successful, it will send back an error message. See pack-protocol.txt +successful, it will send back an error message. See linkgit:gitprotocol-pack[5] for example messages. report-status-v2 @@ -292,7 +304,7 @@ adding new "option" directives in order to support reference rewritten by the "proc-receive" hook. The "proc-receive" hook may handle a command for a pseudo-reference which may create or update a reference with different name, new-oid, and old-oid. While the capability -'report-status' cannot report for such case. See pack-protocol.txt +'report-status' cannot report for such case. See linkgit:gitprotocol-pack[5] for details. delete-refs @@ -378,3 +390,7 @@ packet-line, and must not contain non-printable or whitespace characters. The current implementation uses trace2 session IDs (see link:api-trace2.html[api-trace2] for details), but this may change and users of the session ID should not rely on this fact. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/protocol-common.txt b/Documentation/gitprotocol-common.txt index ecedb34bba..1486651bd1 100644 --- a/Documentation/technical/protocol-common.txt +++ b/Documentation/gitprotocol-common.txt @@ -1,5 +1,20 @@ -Documentation Common to Pack and Http Protocols -=============================================== +gitprotocol-common(5) +===================== + +NAME +---- +gitprotocol-common - Things common to various protocols + +SYNOPSIS +-------- +[verse] +<over-the-wire-protocol> + +DESCRIPTION +----------- + +This document sets defines things common to various over-the-wire +protocols and file formats used in Git. ABNF Notation ------------- @@ -97,3 +112,7 @@ Examples (as C-style strings): "000bfoobar\n" "foobar\n" "0004" "" ---- + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/http-protocol.txt b/Documentation/gitprotocol-http.txt index cc5126cfed..ccc13f0a40 100644 --- a/Documentation/technical/http-protocol.txt +++ b/Documentation/gitprotocol-http.txt @@ -1,5 +1,19 @@ -HTTP transfer protocols -======================= +gitprotocol-http(5) +=================== + +NAME +---- +gitprotocol-http - Git HTTP-based protocols + + +SYNOPSIS +-------- +[verse] +<over-the-wire-protocol> + + +DESCRIPTION +----------- Git supports two HTTP based transfer protocols. A "dumb" protocol which requires only a standard HTTP server on the server end of the @@ -222,7 +236,7 @@ smart server reply: S: 0000 The client may send Extra Parameters (see -Documentation/technical/pack-protocol.txt) as a colon-separated string +linkgit:gitprotocol-pack[5]) as a colon-separated string in the Git-Protocol HTTP header. Uses the `--http-backend-info-refs` option to @@ -512,11 +526,18 @@ the id obtained through ref discovery as old_id. TODO: Document this further. - -References +REFERENCES ---------- http://www.ietf.org/rfc/rfc1738.txt[RFC 1738: Uniform Resource Locators (URL)] http://www.ietf.org/rfc/rfc2616.txt[RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1] -link:technical/pack-protocol.html -link:technical/protocol-capabilities.html + +SEE ALSO +-------- + +linkgit:gitprotocol-pack[5] +linkgit:gitprotocol-capabilities[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/pack-protocol.txt b/Documentation/gitprotocol-pack.txt index e13a2c064d..dd4108b7a3 100644 --- a/Documentation/technical/pack-protocol.txt +++ b/Documentation/gitprotocol-pack.txt @@ -1,11 +1,23 @@ -Packfile transfer protocols -=========================== +gitprotocol-pack(5) +=================== + +NAME +---- +gitprotocol-pack - How packs are transferred over-the-wire + +SYNOPSIS +-------- +[verse] +<over-the-wire-protocol> + +DESCRIPTION +----------- Git supports transferring data in packfiles over the ssh://, git://, http:// and file:// transports. There exist two sets of protocols, one for pushing data from a client to a server and another for fetching data from a server to a client. The three transports (ssh, git, file) use the same -protocol to transfer data. http is documented in http-protocol.txt. +protocol to transfer data. http is documented in linkgit:gitprotocol-http[5]. The processes invoked in the canonical Git implementation are 'upload-pack' on the server side and 'fetch-pack' on the client side for fetching data; @@ -18,7 +30,7 @@ pkt-line Format --------------- The descriptions below build on the pkt-line format described in -protocol-common.txt. When the grammar indicate `PKT-LINE(...)`, unless +linkgit:gitprotocol-common[5]. When the grammar indicate `PKT-LINE(...)`, unless otherwise noted the usual pkt-line LF rules apply: the sender SHOULD include a LF, but the receiver MUST NOT complain if it is not present. @@ -60,7 +72,7 @@ Each Extra Parameter takes the form of `<key>=<value>` or `<key>`. Servers that receive any such Extra Parameters MUST ignore all unrecognized keys. Currently, the only Extra Parameter recognized is -"version" with a value of '1' or '2'. See protocol-v2.txt for more +"version" with a value of '1' or '2'. See linkgit:gitprotocol-v2[5] for more information on protocol version 2. Git Transport @@ -455,7 +467,7 @@ Now that the client and server have finished negotiation about what the minimal amount of data that needs to be sent to the client is, the server will construct and send the required data in packfile format. -See pack-format.txt for what the packfile itself actually looks like. +See linkgit:gitformat-pack[5] for what the packfile itself actually looks like. If 'side-band' or 'side-band-64k' capabilities have been specified by the client, the server will send the packfile data multiplexed. @@ -707,3 +719,7 @@ An example client/server communication might look like this: S: 0018ok refs/heads/debug\n S: 002ang refs/heads/master non-fast-forward\n ---- + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/gitprotocol-v2.txt index 8a877d27e2..c9c0f9160b 100644 --- a/Documentation/technical/protocol-v2.txt +++ b/Documentation/gitprotocol-v2.txt @@ -1,5 +1,17 @@ -Git Wire Protocol, Version 2 -============================ +gitprotocol-v2(5) +================= + +NAME +---- +gitprotocol-v2 - Git Wire Protocol, Version 2 + +SYNOPSIS +-------- +[verse] +<over-the-wire-protocol> + +DESCRIPTION +----------- This document presents a specification for a version 2 of Git's wire protocol. Protocol v2 will improve upon v1 in the following ways: @@ -26,8 +38,7 @@ Packet-Line Framing ------------------- All communication is done using packet-line framing, just as in v1. See -`Documentation/technical/pack-protocol.txt` and -`Documentation/technical/protocol-common.txt` for more information. +linkgit:gitprotocol-pack[5] and linkgit:gitprotocol-common[5] for more information. In protocol v2 these special packets will have the following semantics: @@ -42,7 +53,7 @@ Initial Client Request In general a client can request to speak protocol v2 by sending `version=2` through the respective side-channel for the transport being used which inevitably sets `GIT_PROTOCOL`. More information can be -found in `pack-protocol.txt` and `http-protocol.txt`, as well as the +found in linkgit:gitprotocol-pack[5] and linkgit:gitprotocol-http[5], as well as the `GIT_PROTOCOL` definition in `git.txt`. In all cases the response from the server is the capability advertisement. @@ -66,7 +77,7 @@ HTTP Transport ~~~~~~~~~~~~~~ When using the http:// or https:// transport a client makes a "smart" -info/refs request as described in `http-protocol.txt` and requests that +info/refs request as described in linkgit:gitprotocol-http[5] and requests that v2 be used by supplying "version=2" in the `Git-Protocol` header. C: GET $GIT_URL/info/refs?service=git-upload-pack HTTP/1.0 @@ -566,3 +577,7 @@ and associated requested information, each separated by a single space. attr = "size" obj-info = obj-id SP obj-size + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitremote-helpers.txt b/Documentation/gitremote-helpers.txt index 6f1e269ae4..ed8da428c9 100644 --- a/Documentation/gitremote-helpers.txt +++ b/Documentation/gitremote-helpers.txt @@ -168,6 +168,9 @@ Supported commands: 'list', 'import'. Can guarantee that when a clone is requested, the received pack is self contained and is connected. +'get':: + Can use the 'get' command to download a file from a given URI. + If a helper advertises 'connect', Git will use it if possible and fall back to another capability if the helper requests so when connecting (see the 'connect' command under COMMANDS). @@ -418,6 +421,12 @@ Supported if the helper has the "connect" capability. + Supported if the helper has the "stateless-connect" capability. +'get' <uri> <path>:: + Downloads the file from the given `<uri>` to the given `<path>`. If + `<path>.temp` exists, then Git assumes that the `.temp` file is a + partial download from a previous attempt and will resume the + download from that position. + If a fatal error occurs, the program writes the error message to stderr and exits. The caller should expect that a suitable error message has been printed if the child closes the connection without diff --git a/Documentation/howto/recover-corrupted-object-harder.txt b/Documentation/howto/recover-corrupted-object-harder.txt index 8994e2559e..5efb4fe81f 100644 --- a/Documentation/howto/recover-corrupted-object-harder.txt +++ b/Documentation/howto/recover-corrupted-object-harder.txt @@ -68,7 +68,7 @@ Note that the "object" file isn't fit for feeding straight to zlib; it has the git packed object header, which is variable-length. We want to strip that off so we can start playing with the zlib data directly. You can either work your way through it manually (the format is described in -link:../technical/pack-format.html[Documentation/technical/pack-format.txt]), +linkgit:gitformat-pack[5]), or you can walk through it in a debugger. I did the latter, creating a valid pack like: diff --git a/Documentation/lint-man-section-order.perl b/Documentation/lint-man-section-order.perl index 425377dfeb..02408a0062 100755 --- a/Documentation/lint-man-section-order.perl +++ b/Documentation/lint-man-section-order.perl @@ -32,6 +32,9 @@ my %SECTIONS; 'SEE ALSO' => { order => $order++, }, + 'FILE FORMAT' => { + order => $order++, + }, 'GIT' => { required => 1, order => $order++, diff --git a/Documentation/rerere-options.txt b/Documentation/rerere-options.txt new file mode 100644 index 0000000000..c3321ddea2 --- /dev/null +++ b/Documentation/rerere-options.txt @@ -0,0 +1,9 @@ +--rerere-autoupdate:: +--no-rerere-autoupdate:: + After the rerere mechanism reuses a recorded resolution on + the current conflict to update the files in the working + tree, allow it to also update the index with the result of + resolution. `--no-rerere-autoupdate` is a good way to + double-check what `rerere` did and catch potential + mismerges, before committing the result to the index with a + separate `git add`. diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt index 195e74eec6..1837509566 100644 --- a/Documentation/rev-list-options.txt +++ b/Documentation/rev-list-options.txt @@ -242,6 +242,7 @@ ifdef::git-rev-list[] to `/dev/null` as the output does not have to be formatted. --disk-usage:: +--disk-usage=human:: Suppress normal output; instead, print the sum of the bytes used for on-disk storage by the selected commits or objects. This is equivalent to piping the output into `git cat-file @@ -249,6 +250,8 @@ ifdef::git-rev-list[] faster (especially with `--use-bitmap-index`). See the `CAVEATS` section in linkgit:git-cat-file[1] for the limitations of what "on-disk storage" means. + With the optional value `human`, on-disk storage size is shown + in human-readable string(e.g. 12.24 Kib, 3.50 Mib). endif::git-rev-list[] --cherry-mark:: @@ -389,12 +392,14 @@ Default mode:: merges from the resulting history, as there are no selected commits contributing to this merge. ---ancestry-path:: +--ancestry-path[=<commit>]:: When given a range of commits to display (e.g. 'commit1..commit2' - or 'commit2 {caret}commit1'), only display commits that exist - directly on the ancestry chain between the 'commit1' and - 'commit2', i.e. commits that are both descendants of 'commit1', - and ancestors of 'commit2'. + or 'commit2 {caret}commit1'), only display commits in that range + that are ancestors of <commit>, descendants of <commit>, or + <commit> itself. If no commit is specified, use 'commit1' (the + excluded part of the range) as <commit>. Can be passed multiple + times; if so, a commit is included if it is any of the commits + given or if it is an ancestor or descendant of one of them. A more detailed explanation follows. @@ -568,11 +573,10 @@ Note the major differences in `N`, `P`, and `Q` over `--full-history`: There is another simplification mode available: ---ancestry-path:: - Limit the displayed commits to those directly on the ancestry - chain between the ``from'' and ``to'' commits in the given commit - range. I.e. only display commits that are ancestor of the ``to'' - commit and descendants of the ``from'' commit. +--ancestry-path[=<commit>]:: + Limit the displayed commits to those which are an ancestor of + <commit>, or which are a descendant of <commit>, or are <commit> + itself. + As an example use case, consider the following commit history: + @@ -604,6 +608,29 @@ option does. Applied to the 'D..M' range, it results in: \ L--M ----------------------------------------------------------------------- ++ +We can also use `--ancestry-path=D` instead of `--ancestry-path` which +means the same thing when applied to the 'D..M' range but is just more +explicit. ++ +If we instead are interested in a given topic within this range, and all +commits affected by that topic, we may only want to view the subset of +`D..M` which contain that topic in their ancestry path. So, using +`--ancestry-path=H D..M` for example would result in: ++ +----------------------------------------------------------------------- + E + \ + G---H---I---J + \ + L--M +----------------------------------------------------------------------- ++ +Whereas `--ancestry-path=K D..M` would result in ++ +----------------------------------------------------------------------- + K---------------L--M +----------------------------------------------------------------------- Before discussing another option, `--show-pulls`, we need to create a new example history. @@ -659,7 +686,7 @@ Here, the merge commits `O` and `P` contribute extra noise, as they did not actually contribute a change to `file.txt`. They only merged a topic that was based on an older version of `file.txt`. This is a common issue in repositories using a workflow where many contributors work in -parallel and merge their topic branches along a single trunk: manu +parallel and merge their topic branches along a single trunk: many unrelated merges appear in the `--full-history` results. When using the `--simplify-merges` option, the commits `O` and `P` diff --git a/Documentation/technical/api-simple-ipc.txt b/Documentation/technical/api-simple-ipc.txt index d79ad323e6..d44ada98e7 100644 --- a/Documentation/technical/api-simple-ipc.txt +++ b/Documentation/technical/api-simple-ipc.txt @@ -78,7 +78,7 @@ client and an optional response message from the server. Both the client and server messages are unlimited in length and are terminated with a flush packet. -The pkt-line routines (Documentation/technical/protocol-common.txt) +The pkt-line routines (linkgit:gitprotocol-common[5]) are used to simplify buffer management during message generation, transmission, and reception. A flush packet is used to mark the end of the message. This allows the sender to incrementally generate and diff --git a/Documentation/technical/api-trace2.txt b/Documentation/technical/api-trace2.txt index 77a150b30e..2afa28bb5a 100644 --- a/Documentation/technical/api-trace2.txt +++ b/Documentation/technical/api-trace2.txt @@ -717,6 +717,7 @@ The "exec_id" field is a command-unique id and is only useful if the { "event":"def_param", ... + "scope":"global", "param":"core.abbrev", "value":"7" } @@ -1207,6 +1208,45 @@ at offset 508. This example also shows that thread names are assigned in a racy manner as each thread starts and allocates TLS storage. +Config (def param) Events:: + + Dump "interesting" config values to trace2 log. ++ +We can optionally emit configuration events, see +`trace2.configparams` in linkgit:git-config[1] for how to enable +it. ++ +---------------- +$ git config --system color.ui never +$ git config --global color.ui always +$ git config --local color.ui auto +$ git config --list --show-scope | grep 'color.ui' +system color.ui=never +global color.ui=always +local color.ui=auto +---------------- ++ +Then, mark the config `color.ui` as "interesting" config with +`GIT_TRACE2_CONFIG_PARAMS`: ++ +---------------- +$ export GIT_TRACE2_PERF_BRIEF=1 +$ export GIT_TRACE2_PERF=~/log.perf +$ export GIT_TRACE2_CONFIG_PARAMS=color.ui +$ git version +... +$ cat ~/log.perf +d0 | main | version | | | | | ... +d0 | main | start | | 0.001642 | | | /usr/local/bin/git version +d0 | main | cmd_name | | | | | version (version) +d0 | main | def_param | | | | scope:system | color.ui:never +d0 | main | def_param | | | | scope:global | color.ui:always +d0 | main | def_param | | | | scope:local | color.ui:auto +d0 | main | data | r0 | 0.002100 | 0.002100 | fsync | fsync/writeout-only:0 +d0 | main | data | r0 | 0.002126 | 0.002126 | fsync | fsync/hardware-flush:0 +d0 | main | exit | | 0.000470 | | | code:0 +d0 | main | atexit | | 0.000477 | | | code:0 +---------------- == Future Work === Relationship to the Existing Trace Api (api-trace.txt) diff --git a/Documentation/technical/bundle-uri.txt b/Documentation/technical/bundle-uri.txt new file mode 100644 index 0000000000..c25c42378a --- /dev/null +++ b/Documentation/technical/bundle-uri.txt @@ -0,0 +1,573 @@ +Bundle URIs +=========== + +Git bundles are files that store a pack-file along with some extra metadata, +including a set of refs and a (possibly empty) set of necessary commits. See +linkgit:git-bundle[1] and link:bundle-format.txt[the bundle format] for more +information. + +Bundle URIs are locations where Git can download one or more bundles in +order to bootstrap the object database in advance of fetching the remaining +objects from a remote. + +One goal is to speed up clones and fetches for users with poor network +connectivity to the origin server. Another benefit is to allow heavy users, +such as CI build farms, to use local resources for the majority of Git data +and thereby reducing the load on the origin server. + +To enable the bundle URI feature, users can specify a bundle URI using +command-line options or the origin server can advertise one or more URIs +via a protocol v2 capability. + +Design Goals +------------ + +The bundle URI standard aims to be flexible enough to satisfy multiple +workloads. The bundle provider and the Git client have several choices in +how they create and consume bundle URIs. + +* Bundles can have whatever name the server desires. This name could refer + to immutable data by using a hash of the bundle contents. However, this + means that a new URI will be needed after every update of the content. + This might be acceptable if the server is advertising the URI (and the + server is aware of new bundles being generated) but would not be + ergonomic for users using the command line option. + +* The bundles could be organized specifically for bootstrapping full + clones, but could also be organized with the intention of bootstrapping + incremental fetches. The bundle provider must decide on one of several + organization schemes to minimize client downloads during incremental + fetches, but the Git client can also choose whether to use bundles for + either of these operations. + +* The bundle provider can choose to support full clones, partial clones, + or both. The client can detect which bundles are appropriate for the + repository's partial clone filter, if any. + +* The bundle provider can use a single bundle (for clones only), or a + list of bundles. When using a list of bundles, the provider can specify + whether or not the client needs _all_ of the bundle URIs for a full + clone, or if _any_ one of the bundle URIs is sufficient. This allows the + bundle provider to use different URIs for different geographies. + +* The bundle provider can organize the bundles using heuristics, such as + creation tokens, to help the client prevent downloading bundles it does + not need. When the bundle provider does not provide these heuristics, + the client can use optimizations to minimize how much of the data is + downloaded. + +* The bundle provider does not need to be associated with the Git server. + The client can choose to use the bundle provider without it being + advertised by the Git server. + +* The client can choose to discover bundle providers that are advertised + by the Git server. This could happen during `git clone`, during + `git fetch`, both, or neither. The user can choose which combination + works best for them. + +* The client can choose to configure a bundle provider manually at any + time. The client can also choose to specify a bundle provider manually + as a command-line option to `git clone`. + +Each repository is different and every Git server has different needs. +Hopefully the bundle URI feature is flexible enough to satisfy all needs. +If not, then the feature can be extended through its versioning mechanism. + +Server requirements +------------------- + +To provide a server-side implementation of bundle servers, no other parts +of the Git protocol are required. This allows server maintainers to use +static content solutions such as CDNs in order to serve the bundle files. + +At the current scope of the bundle URI feature, all URIs are expected to +be HTTP(S) URLs where content is downloaded to a local file using a `GET` +request to that URL. The server could include authentication requirements +to those requests with the aim of triggering the configured credential +helper for secure access. (Future extensions could use "file://" URIs or +SSH URIs.) + +Assuming a `200 OK` response from the server, the content at the URL is +inspected. First, Git attempts to parse the file as a bundle file of +version 2 or higher. If the file is not a bundle, then the file is parsed +as a plain-text file using Git's config parser. The key-value pairs in +that config file are expected to describe a list of bundle URIs. If +neither of these parse attempts succeed, then Git will report an error to +the user that the bundle URI provided erroneous data. + +Any other data provided by the server is considered erroneous. + +Bundle Lists +------------ + +The Git server can advertise bundle URIs using a set of `key=value` pairs. +A bundle URI can also serve a plain-text file in the Git config format +containing these same `key=value` pairs. In both cases, we consider this +to be a _bundle list_. The pairs specify information about the bundles +that the client can use to make decisions for which bundles to download +and which to ignore. + +A few keys focus on properties of the list itself. + +bundle.version:: + (Required) This value provides a version number for the bundle + list. If a future Git change enables a feature that needs the Git + client to react to a new key in the bundle list file, then this version + will increment. The only current version number is 1, and if any other + value is specified then Git will fail to use this file. + +bundle.mode:: + (Required) This value has one of two values: `all` and `any`. When `all` + is specified, then the client should expect to need all of the listed + bundle URIs that match their repository's requirements. When `any` is + specified, then the client should expect that any one of the bundle URIs + that match their repository's requirements will suffice. Typically, the + `any` option is used to list a number of different bundle servers + located in different geographies. + +bundle.heuristic:: + If this string-valued key exists, then the bundle list is designed to + work well with incremental `git fetch` commands. The heuristic signals + that there are additional keys available for each bundle that help + determine which subset of bundles the client should download. The only + heuristic currently planned is `creationToken`. + +The remaining keys include an `<id>` segment which is a server-designated +name for each available bundle. The `<id>` must contain only alphanumeric +and `-` characters. + +bundle.<id>.uri:: + (Required) This string value is the URI for downloading bundle `<id>`. + If the URI begins with a protocol (`http://` or `https://`) then the URI + is absolute. Otherwise, the URI is interpreted as relative to the URI + used for the bundle list. If the URI begins with `/`, then that relative + path is relative to the domain name used for the bundle list. (This use + of relative paths is intended to make it easier to distribute a set of + bundles across a large number of servers or CDNs with different domain + names.) + +bundle.<id>.filter:: + This string value represents an object filter that should also appear in + the header of this bundle. The server uses this value to differentiate + different kinds of bundles from which the client can choose those that + match their object filters. + +bundle.<id>.creationToken:: + This value is a nonnegative 64-bit integer used for sorting the bundles + the list. This is used to download a subset of bundles during a fetch + when `bundle.heuristic=creationToken`. + +bundle.<id>.location:: + This string value advertises a real-world location from where the bundle + URI is served. This can be used to present the user with an option for + which bundle URI to use or simply as an informative indicator of which + bundle URI was selected by Git. This is only valuable when + `bundle.mode` is `any`. + +Here is an example bundle list using the Git config format: + + [bundle] + version = 1 + mode = all + heuristic = creationToken + + [bundle "2022-02-09-1644442601-daily"] + uri = https://bundles.example.com/git/git/2022-02-09-1644442601-daily.bundle + creationToken = 1644442601 + + [bundle "2022-02-02-1643842562"] + uri = https://bundles.example.com/git/git/2022-02-02-1643842562.bundle + creationToken = 1643842562 + + [bundle "2022-02-09-1644442631-daily-blobless"] + uri = 2022-02-09-1644442631-daily-blobless.bundle + creationToken = 1644442631 + filter = blob:none + + [bundle "2022-02-02-1643842568-blobless"] + uri = /git/git/2022-02-02-1643842568-blobless.bundle + creationToken = 1643842568 + filter = blob:none + +This example uses `bundle.mode=all` as well as the +`bundle.<id>.creationToken` heuristic. It also uses the `bundle.<id>.filter` +options to present two parallel sets of bundles: one for full clones and +another for blobless partial clones. + +Suppose that this bundle list was found at the URI +`https://bundles.example.com/git/git/` and so the two blobless bundles have +the following fully-expanded URIs: + +* `https://bundles.example.com/git/git/2022-02-09-1644442631-daily-blobless.bundle` +* `https://bundles.example.com/git/git/2022-02-02-1643842568-blobless.bundle` + +Advertising Bundle URIs +----------------------- + +If a user knows a bundle URI for the repository they are cloning, then +they can specify that URI manually through a command-line option. However, +a Git host may want to advertise bundle URIs during the clone operation, +helping users unaware of the feature. + +The only thing required for this feature is that the server can advertise +one or more bundle URIs. This advertisement takes the form of a new +protocol v2 capability specifically for discovering bundle URIs. + +The client could choose an arbitrary bundle URI as an option _or_ select +the URI with best performance by some exploratory checks. It is up to the +bundle provider to decide if having multiple URIs is preferable to a +single URI that is geodistributed through server-side infrastructure. + +Cloning with Bundle URIs +------------------------ + +The primary need for bundle URIs is to speed up clones. The Git client +will interact with bundle URIs according to the following flow: + +1. The user specifies a bundle URI with the `--bundle-uri` command-line + option _or_ the client discovers a bundle list advertised by the + Git server. + +2. If the downloaded data from a bundle URI is a bundle, then the client + inspects the bundle headers to check that the prerequisite commit OIDs + are present in the client repository. If some are missing, then the + client delays unbundling until other bundles have been unbundled, + making those OIDs present. When all required OIDs are present, the + client unbundles that data using a refspec. The default refspec is + `+refs/heads/*:refs/bundles/*`, but this can be configured. These refs + are stored so that later `git fetch` negotiations can communicate the + bundled refs as `have`s, reducing the size of the fetch over the Git + protocol. To allow pruning refs from this ref namespace, Git may + introduce a numbered namespace (such as `refs/bundles/<i>/*`) such that + stale bundle refs can be deleted. + +3. If the file is instead a bundle list, then the client inspects the + `bundle.mode` to see if the list is of the `all` or `any` form. + + a. If `bundle.mode=all`, then the client considers all bundle + URIs. The list is reduced based on the `bundle.<id>.filter` options + matching the client repository's partial clone filter. Then, all + bundle URIs are requested. If the `bundle.<id>.creationToken` + heuristic is provided, then the bundles are downloaded in decreasing + order by the creation token, stopping when a bundle has all required + OIDs. The bundles can then be unbundled in increasing creation token + order. The client stores the latest creation token as a heuristic + for avoiding future downloads if the bundle list does not advertise + bundles with larger creation tokens. + + b. If `bundle.mode=any`, then the client can choose any one of the + bundle URIs to inspect. The client can use a variety of ways to + choose among these URIs. The client can also fallback to another URI + if the initial choice fails to return a result. + +Note that during a clone we expect that all bundles will be required, and +heuristics such as `bundle.<uri>.creationToken` can be used to download +bundles in chronological order or in parallel. + +If a given bundle URI is a bundle list with a `bundle.heuristic` +value, then the client can choose to store that URI as its chosen bundle +URI. The client can then navigate directly to that URI during later `git +fetch` calls. + +When downloading bundle URIs, the client can choose to inspect the initial +content before committing to downloading the entire content. This may +provide enough information to determine if the URI is a bundle list or +a bundle. In the case of a bundle, the client may inspect the bundle +header to determine that all advertised tips are already in the client +repository and cancel the remaining download. + +Fetching with Bundle URIs +------------------------- + +When the client fetches new data, it can decide to fetch from bundle +servers before fetching from the origin remote. This could be done via a +command-line option, but it is more likely useful to use a config value +such as the one specified during the clone. + +The fetch operation follows the same procedure to download bundles from a +bundle list (although we do _not_ want to use parallel downloads here). We +expect that the process will end when all prerequisite commit OIDs in a +thin bundle are already in the object database. + +When using the `creationToken` heuristic, the client can avoid downloading +any bundles if their creation tokenss are not larger than the stored +creation token. After fetching new bundles, Git updates this local +creation token. + +If the bundle provider does not provide a heuristic, then the client +should attempt to inspect the bundle headers before downloading the full +bundle data in case the bundle tips already exist in the client +repository. + +Error Conditions +---------------- + +If the Git client discovers something unexpected while downloading +information according to a bundle URI or the bundle list found at that +location, then Git can ignore that data and continue as if it was not +given a bundle URI. The remote Git server is the ultimate source of truth, +not the bundle URI. + +Here are a few example error conditions: + +* The client fails to connect with a server at the given URI or a connection + is lost without any chance to recover. + +* The client receives a 400-level response (such as `404 Not Found` or + `401 Not Authorized`). The client should use the credential helper to + find and provide a credential for the URI, but match the semantics of + Git's other HTTP protocols in terms of handling specific 400-level + errors. + +* The server reports any other failure reponse. + +* The client receives data that is not parsable as a bundle or bundle list. + +* A bundle includes a filter that does not match expectations. + +* The client cannot unbundle the bundles because the prerequisite commit OIDs + are not in the object database and there are no more bundles to download. + +There are also situations that could be seen as wasteful, but are not +error conditions: + +* The downloaded bundles contain more information than is requested by + the clone or fetch request. A primary example is if the user requests + a clone with `--single-branch` but downloads bundles that store every + reachable commit from all `refs/heads/*` references. This might be + initially wasteful, but perhaps these objects will become reachable by + a later ref update that the client cares about. + +* A bundle download during a `git fetch` contains objects already in the + object database. This is probably unavoidable if we are using bundles + for fetches, since the client will almost always be slightly ahead of + the bundle servers after performing its "catch-up" fetch to the remote + server. This extra work is most wasteful when the client is fetching + much more frequently than the server is computing bundles, such as if + the client is using hourly prefetches with background maintenance, but + the server is computing bundles weekly. For this reason, the client + should not use bundle URIs for fetch unless the server has explicitly + recommended it through a `bundle.heuristic` value. + +Example Bundle Provider organization +------------------------------------ + +The bundle URI feature is intentionally designed to be flexible to +different ways a bundle provider wants to organize the object data. +However, it can be helpful to have a complete organization model described +here so providers can start from that base. + +This example organization is a simplified model of what is used by the +GVFS Cache Servers (see section near the end of this document) which have +been beneficial in speeding up clones and fetches for very large +repositories, although using extra software outside of Git. + +The bundle provider deploys servers across multiple geographies. Each +server manages its own bundle set. The server can track a number of Git +repositories, but provides a bundle list for each based on a pattern. For +example, when mirroring a repository at `https://<domain>/<org>/<repo>` +the bundle server could have its bundle list available at +`https://<server-url>/<domain>/<org>/<repo>`. The origin Git server can +list all of these servers under the "any" mode: + + [bundle] + version = 1 + mode = any + + [bundle "eastus"] + uri = https://eastus.example.com/<domain>/<org>/<repo> + + [bundle "europe"] + uri = https://europe.example.com/<domain>/<org>/<repo> + + [bundle "apac"] + uri = https://apac.example.com/<domain>/<org>/<repo> + +This "list of lists" is static and only changes if a bundle server is +added or removed. + +Each bundle server manages its own set of bundles. The initial bundle list +contains only a single bundle, containing all of the objects received from +cloning the repository from the origin server. The list uses the +`creationToken` heuristic and a `creationToken` is made for the bundle +based on the server's timestamp. + +The bundle server runs regularly-scheduled updates for the bundle list, +such as once a day. During this task, the server fetches the latest +contents from the origin server and generates a bundle containing the +objects reachable from the latest origin refs, but not contained in a +previously-computed bundle. This bundle is added to the list, with care +that the `creationToken` is strictly greater than the previous maximum +`creationToken`. + +When the bundle list grows too large, say more than 30 bundles, then the +oldest "_N_ minus 30" bundles are combined into a single bundle. This +bundle's `creationToken` is equal to the maximum `creationToken` among the +merged bundles. + +An example bundle list is provided here, although it only has two daily +bundles and not a full list of 30: + + [bundle] + version = 1 + mode = all + heuristic = creationToken + + [bundle "2022-02-13-1644770820-daily"] + uri = https://eastus.example.com/<domain>/<org>/<repo>/2022-02-09-1644770820-daily.bundle + creationToken = 1644770820 + + [bundle "2022-02-09-1644442601-daily"] + uri = https://eastus.example.com/<domain>/<org>/<repo>/2022-02-09-1644442601-daily.bundle + creationToken = 1644442601 + + [bundle "2022-02-02-1643842562"] + uri = https://eastus.example.com/<domain>/<org>/<repo>/2022-02-02-1643842562.bundle + creationToken = 1643842562 + +To avoid storing and serving object data in perpetuity despite becoming +unreachable in the origin server, this bundle merge can be more careful. +Instead of taking an absolute union of the old bundles, instead the bundle +can be created by looking at the newer bundles and ensuring that their +necessary commits are all available in this merged bundle (or in another +one of the newer bundles). This allows "expiring" object data that is not +being used by new commits in this window of time. That data could be +reintroduced by a later push. + +The intention of this data organization has two main goals. First, initial +clones of the repository become faster by downloading precomputed object +data from a closer source. Second, `git fetch` commands can be faster, +especially if the client has not fetched for a few days. However, if a +client does not fetch for 30 days, then the bundle list organization would +cause redownloading a large amount of object data. + +One way to make this organization more useful to users who fetch frequently +is to have more frequent bundle creation. For example, bundles could be +created every hour, and then once a day those "hourly" bundles could be +merged into a "daily" bundle. The daily bundles are merged into the +oldest bundle after 30 days. + +It is recommened that this bundle strategy is repeated with the `blob:none` +filter if clients of this repository are expecting to use blobless partial +clones. This list of blobless bundles stays in the same list as the full +bundles, but uses the `bundle.<id>.filter` key to separate the two groups. +For very large repositories, the bundle provider may want to _only_ provide +blobless bundles. + +Implementation Plan +------------------- + +This design document is being submitted on its own as an aspirational +document, with the goal of implementing all of the mentioned client +features over the course of several patch series. Here is a potential +outline for submitting these features: + +1. Integrate bundle URIs into `git clone` with a `--bundle-uri` option. + This will include a new `git fetch --bundle-uri` mode for use as the + implementation underneath `git clone`. The initial version here will + expect a single bundle at the given URI. + +2. Implement the ability to parse a bundle list from a bundle URI and + update the `git fetch --bundle-uri` logic to properly distinguish + between `bundle.mode` options. Specifically design the feature so + that the config format parsing feeds a list of key-value pairs into the + bundle list logic. + +3. Create the `bundle-uri` protocol v2 command so Git servers can advertise + bundle URIs using the key-value pairs. Plug into the existing key-value + input to the bundle list logic. Allow `git clone` to discover these + bundle URIs and bootstrap the client repository from the bundle data. + (This choice is an opt-in via a config option and a command-line + option.) + +4. Allow the client to understand the `bundle.flag=forFetch` configuration + and the `bundle.<id>.creationToken` heuristic. When `git clone` + discovers a bundle URI with `bundle.flag=forFetch`, it configures the + client repository to check that bundle URI during later `git fetch <remote>` + commands. + +5. Allow clients to discover bundle URIs during `git fetch` and configure + a bundle URI for later fetches if `bundle.flag=forFetch`. + +6. Implement the "inspect headers" heuristic to reduce data downloads when + the `bundle.<id>.creationToken` heuristic is not available. + +As these features are reviewed, this plan might be updated. We also expect +that new designs will be discovered and implemented as this feature +matures and becomes used in real-world scenarios. + +Related Work: Packfile URIs +--------------------------- + +The Git protocol already has a capability where the Git server can list +a set of URLs along with the packfile response when serving a client +request. The client is then expected to download the packfiles at those +locations in order to have a complete understanding of the response. + +This mechanism is used by the Gerrit server (implemented with JGit) and +has been effective at reducing CPU load and improving user performance for +clones. + +A major downside to this mechanism is that the origin server needs to know +_exactly_ what is in those packfiles, and the packfiles need to be available +to the user for some time after the server has responded. This coupling +between the origin and the packfile data is difficult to manage. + +Further, this implementation is extremely hard to make work with fetches. + +Related Work: GVFS Cache Servers +-------------------------------- + +The GVFS Protocol [2] is a set of HTTP endpoints designed independently of +the Git project before Git's partial clone was created. One feature of this +protocol is the idea of a "cache server" which can be colocated with build +machines or developer offices to transfer Git data without overloading the +central server. + +The endpoint that VFS for Git is famous for is the `GET /gvfs/objects/{oid}` +endpoint, which allows downloading an object on-demand. This is a critical +piece of the filesystem virtualization of that product. + +However, a more subtle need is the `GET /gvfs/prefetch?lastPackTimestamp=<t>` +endpoint. Given an optional timestamp, the cache server responds with a list +of precomputed packfiles containing the commits and trees that were introduced +in those time intervals. + +The cache server computes these "prefetch" packfiles using the following +strategy: + +1. Every hour, an "hourly" pack is generated with a given timestamp. +2. Nightly, the previous 24 hourly packs are rolled up into a "daily" pack. +3. Nightly, all prefetch packs more than 30 days old are rolled up into + one pack. + +When a user runs `gvfs clone` or `scalar clone` against a repo with cache +servers, the client requests all prefetch packfiles, which is at most +`24 + 30 + 1` packfiles downloading only commits and trees. The client +then follows with a request to the origin server for the references, and +attempts to checkout that tip reference. (There is an extra endpoint that +helps get all reachable trees from a given commit, in case that commit +was not already in a prefetch packfile.) + +During a `git fetch`, a hook requests the prefetch endpoint using the +most-recent timestamp from a previously-downloaded prefetch packfile. +Only the list of packfiles with later timestamps are downloaded. Most +users fetch hourly, so they get at most one hourly prefetch pack. Users +whose machines have been off or otherwise have not fetched in over 30 days +might redownload all prefetch packfiles. This is rare. + +It is important to note that the clients always contact the origin server +for the refs advertisement, so the refs are frequently "ahead" of the +prefetched pack data. The missing objects are downloaded on-demand using +the `GET gvfs/objects/{oid}` requests, when needed by a command such as +`git checkout` or `git log`. Some Git optimizations disable checks that +would cause these on-demand downloads to be too aggressive. + +See Also +-------- + +[1] https://lore.kernel.org/git/RFC-cover-00.13-0000000000-20210805T150534Z-avarab@gmail.com/ + An earlier RFC for a bundle URI feature. + +[2] https://github.com/microsoft/VFSForGit/blob/master/Protocol.md + The GVFS Protocol diff --git a/Documentation/technical/cruft-packs.txt b/Documentation/technical/cruft-packs.txt deleted file mode 100644 index d81f3a8982..0000000000 --- a/Documentation/technical/cruft-packs.txt +++ /dev/null @@ -1,123 +0,0 @@ -= Cruft packs - -The cruft packs feature offer an alternative to Git's traditional mechanism of -removing unreachable objects. This document provides an overview of Git's -pruning mechanism, and how a cruft pack can be used instead to accomplish the -same. - -== Background - -To remove unreachable objects from your repository, Git offers `git repack -Ad` -(see linkgit:git-repack[1]). Quoting from the documentation: - -[quote] -[...] unreachable objects in a previous pack become loose, unpacked objects, -instead of being left in the old pack. [...] loose unreachable objects will be -pruned according to normal expiry rules with the next 'git gc' invocation. - -Unreachable objects aren't removed immediately, since doing so could race with -an incoming push which may reference an object which is about to be deleted. -Instead, those unreachable objects are stored as loose objects and stay that way -until they are older than the expiration window, at which point they are removed -by linkgit:git-prune[1]. - -Git must store these unreachable objects loose in order to keep track of their -per-object mtimes. If these unreachable objects were written into one big pack, -then either freshening that pack (because an object contained within it was -re-written) or creating a new pack of unreachable objects would cause the pack's -mtime to get updated, and the objects within it would never leave the expiration -window. Instead, objects are stored loose in order to keep track of the -individual object mtimes and avoid a situation where all cruft objects are -freshened at once. - -This can lead to undesirable situations when a repository contains many -unreachable objects which have not yet left the grace period. Having large -directories in the shards of `.git/objects` can lead to decreased performance in -the repository. But given enough unreachable objects, this can lead to inode -starvation and degrade the performance of the whole system. Since we -can never pack those objects, these repositories often take up a large amount of -disk space, since we can only zlib compress them, but not store them in delta -chains. - -== Cruft packs - -A cruft pack eliminates the need for storing unreachable objects in a loose -state by including the per-object mtimes in a separate file alongside a single -pack containing all loose objects. - -A cruft pack is written by `git repack --cruft` when generating a new pack. -linkgit:git-pack-objects[1]'s `--cruft` option. Note that `git repack --cruft` -is a classic all-into-one repack, meaning that everything in the resulting pack is -reachable, and everything else is unreachable. Once written, the `--cruft` -option instructs `git repack` to generate another pack containing only objects -not packed in the previous step (which equates to packing all unreachable -objects together). This progresses as follows: - - 1. Enumerate every object, marking any object which is (a) not contained in a - kept-pack, and (b) whose mtime is within the grace period as a traversal - tip. - - 2. Perform a reachability traversal based on the tips gathered in the previous - step, adding every object along the way to the pack. - - 3. Write the pack out, along with a `.mtimes` file that records the per-object - timestamps. - -This mode is invoked internally by linkgit:git-repack[1] when instructed to -write a cruft pack. Crucially, the set of in-core kept packs is exactly the set -of packs which will not be deleted by the repack; in other words, they contain -all of the repository's reachable objects. - -When a repository already has a cruft pack, `git repack --cruft` typically only -adds objects to it. An exception to this is when `git repack` is given the -`--cruft-expiration` option, which allows the generated cruft pack to omit -expired objects instead of waiting for linkgit:git-gc[1] to expire those objects -later on. - -It is linkgit:git-gc[1] that is typically responsible for removing expired -unreachable objects. - -== Caution for mixed-version environments - -Repositories that have cruft packs in them will continue to work with any older -version of Git. Note, however, that previous versions of Git which do not -understand the `.mtimes` file will use the cruft pack's mtime as the mtime for -all of the objects in it. In other words, do not expect older (pre-cruft pack) -versions of Git to interpret or even read the contents of the `.mtimes` file. - -Note that having mixed versions of Git GC-ing the same repository can lead to -unreachable objects never being completely pruned. This can happen under the -following circumstances: - - - An older version of Git running GC explodes the contents of an existing - cruft pack loose, using the cruft pack's mtime. - - A newer version running GC collects those loose objects into a cruft pack, - where the .mtime file reflects the loose object's actual mtimes, but the - cruft pack mtime is "now". - -Repeating this process will lead to unreachable objects not getting pruned as a -result of repeatedly resetting the objects' mtimes to the present time. - -If you are GC-ing repositories in a mixed version environment, consider omitting -the `--cruft` option when using linkgit:git-repack[1] and linkgit:git-gc[1], and -leaving the `gc.cruftPacks` configuration unset until all writers understand -cruft packs. - -== Alternatives - -Notable alternatives to this design include: - - - The location of the per-object mtime data, and - - Storing unreachable objects in multiple cruft packs. - -On the location of mtime data, a new auxiliary file tied to the pack was chosen -to avoid complicating the `.idx` format. If the `.idx` format were ever to gain -support for optional chunks of data, it may make sense to consolidate the -`.mtimes` format into the `.idx` itself. - -Storing unreachable objects among multiple cruft packs (e.g., creating a new -cruft pack during each repacking operation including only unreachable objects -which aren't already stored in an earlier cruft pack) is significantly more -complicated to construct, and so aren't pursued here. The obvious drawback to -the current implementation is that the entire cruft pack must be re-written from -scratch. diff --git a/Documentation/technical/hash-function-transition.txt b/Documentation/technical/hash-function-transition.txt index 260224b033..e2ac36dd21 100644 --- a/Documentation/technical/hash-function-transition.txt +++ b/Documentation/technical/hash-function-transition.txt @@ -205,7 +205,7 @@ SHA-1 content. Object storage ~~~~~~~~~~~~~~ Loose objects use zlib compression and packed objects use the packed -format described in Documentation/technical/pack-format.txt, just like +format described in linkgit:gitformat-pack[5], just like today. The content that is compressed and stored uses SHA-256 content instead of SHA-1 content. diff --git a/Documentation/technical/long-running-process-protocol.txt b/Documentation/technical/long-running-process-protocol.txt index aa0aa9af1c..6f33654b42 100644 --- a/Documentation/technical/long-running-process-protocol.txt +++ b/Documentation/technical/long-running-process-protocol.txt @@ -3,7 +3,7 @@ Long-running process protocol This protocol is used when Git needs to communicate with an external process throughout the entire life of a single Git command. All -communication is in pkt-line format (see technical/protocol-common.txt) +communication is in pkt-line format (see linkgit:gitprotocol-common[5]) over standard input and standard output. Handshake diff --git a/Documentation/technical/packfile-uri.txt b/Documentation/technical/packfile-uri.txt index 1eb525fe76..9d453d4765 100644 --- a/Documentation/technical/packfile-uri.txt +++ b/Documentation/technical/packfile-uri.txt @@ -18,7 +18,7 @@ a `packfile-uris` argument, the server MAY send a `packfile-uris` section directly before the `packfile` section (right after `wanted-refs` if it is sent) containing URIs of any of the given protocols. The URIs point to packfiles that use only features that the client has declared that it supports -(e.g. ofs-delta and thin-pack). See protocol-v2.txt for the documentation of +(e.g. ofs-delta and thin-pack). See linkgit:gitprotocol-v2[5] for the documentation of this section. Clients should then download and index all the given URIs (in addition to diff --git a/Documentation/technical/partial-clone.txt b/Documentation/technical/partial-clone.txt index 99f0eb3040..92fcee2bff 100644 --- a/Documentation/technical/partial-clone.txt +++ b/Documentation/technical/partial-clone.txt @@ -79,7 +79,7 @@ Design Details upload-pack negotiation. + This uses the existing capability discovery mechanism. -See "filter" in Documentation/technical/pack-protocol.txt. +See "filter" in linkgit:gitprotocol-pack[5]. - Clients pass a "filter-spec" to clone and fetch which is passed to the server to request filtering during packfile construction. diff --git a/Documentation/technical/scalar.txt b/Documentation/technical/scalar.txt new file mode 100644 index 0000000000..0600150b3a --- /dev/null +++ b/Documentation/technical/scalar.txt @@ -0,0 +1,127 @@ +Scalar +====== + +Scalar is a repository management tool that optimizes Git for use in large +repositories. It accomplishes this by helping users to take advantage of +advanced performance features in Git. Unlike most other Git built-in commands, +Scalar is not executed as a subcommand of 'git'; rather, it is built as a +separate executable containing its own series of subcommands. + +Background +---------- + +Scalar was originally designed as an add-on to Git and implemented as a .NET +Core application. It was created based on the learnings from the VFS for Git +project (another application aimed at improving the experience of working with +large repositories). As part of its initial implementation, Scalar relied on +custom features in the Microsoft fork of Git that have since been integrated +into core Git: + +* partial clone, +* commit graphs, +* multi-pack index, +* sparse checkout (cone mode), +* scheduled background maintenance, +* etc + +With the requisite Git functionality in place and a desire to bring the benefits +of Scalar to the larger Git community, the Scalar application itself was ported +from C# to C and integrated upstream. + +Features +-------- + +Scalar is comprised of two major pieces of functionality: automatically +configuring built-in Git performance features and managing repository +enlistments. + +The Git performance features configured by Scalar (see "Background" for +examples) confer substantial performance benefits to large repositories, but are +either too experimental to enable for all of Git yet, or only benefit large +repositories. As new features are introduced, Scalar should be updated +accordingly to incorporate them. This will prevent the tool from becoming stale +while also providing a path for more easily bringing features to the appropriate +users. + +Enlistments are how Scalar knows which repositories on a user's system should +utilize Scalar-configured features. This allows it to update performance +settings when new ones are added to the tool, as well as centrally manage +repository maintenance. The enlistment structure - a root directory with a +`src/` subdirectory containing the cloned repository itself - is designed to +encourage users to route build outputs outside of the repository to avoid the +performance-limiting overhead of ignoring those files in Git. + +Design +------ + +Scalar is implemented in C and interacts with Git via a mix of child process +invocations of Git and direct usage of `libgit.a`. Internally, it is structured +much like other built-ins with subcommands (e.g., `git stash`), containing a +`cmd_<subcommand>()` function for each subcommand, routed through a `cmd_main()` +function. Most options are unique to each subcommand, with `scalar` respecting +some "global" `git` options (e.g., `-c` and `-C`). + +Because `scalar` is not invoked as a Git subcommand (like `git scalar`), it is +built and installed as its own executable in the `bin/` directory, alongside +`git`, `git-gui`, etc. + +Roadmap +------- + +NOTE: this section will be removed once the remaining tasks outlined in this +roadmap are complete. + +Scalar is a large enough project that it is being upstreamed incrementally, +living in `contrib/` until it is feature-complete. So far, the following patch +series have been accepted: + +- `scalar-the-beginning`: The initial patch series which sets up + `contrib/scalar/` and populates it with a minimal `scalar` command that + demonstrates the fundamental ideas. + +- `scalar-c-and-C`: The `scalar` command learns about two options that can be + specified before the command, `-c <key>=<value>` and `-C <directory>`. + +- `scalar-diagnose`: The `scalar` command is taught the `diagnose` subcommand. + +- `scalar-generalize-diagnose`: Move the functionality of `scalar diagnose` + into `git diagnose` and `git bugreport --diagnose`. + +- 'scalar-add-fsmonitor: Enable the built-in FSMonitor in Scalar + enlistments. At the end of this series, Scalar should be feature-complete + from the perspective of a user. + +Roughly speaking (and subject to change), the following series are needed to +"finish" this initial version of Scalar: + +- Move Scalar to toplevel: Move Scalar out of `contrib/` and into the root of + `git`. This includes a variety of related updates, including: + - building & installing Scalar in the Git root-level 'make [install]'. + - builing & testing Scalar as part of CI. + - moving and expanding test coverage of Scalar (including perf tests). + - implementing 'scalar help'/'git help scalar' to display scalar + documentation. + +Finally, there are two additional patch series that exist in Microsoft's fork of +Git, but there is no current plan to upstream them. There are some interesting +ideas there, but the implementation is too specific to Azure Repos and/or VFS +for Git to be of much help in general. + +These still exist mainly because the GVFS protocol is what Azure Repos has +instead of partial clone, while Git is focused on improving partial clone: + +- `scalar-with-gvfs`: The primary purpose of this patch series is to support + existing Scalar users whose repositories are hosted in Azure Repos (which does + not support Git's partial clones, but supports its predecessor, the GVFS + protocol, which is used by Scalar to emulate the partial clone). + + Since the GVFS protocol will never be supported by core Git, this patch series + will remain in Microsoft's fork of Git. + +- `run-scalar-functional-tests`: The Scalar project developed a quite + comprehensive set of integration tests (or, "Functional Tests"). They are the + sole remaining part of the original C#-based Scalar project, and this patch + adds a GitHub workflow that runs them all. + + Since the tests partially depend on features that are only provided in the + `scalar-with-gvfs` patch series, this patch cannot be upstreamed. diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt index 865074bed4..ca9decdd95 100644 --- a/Documentation/user-manual.txt +++ b/Documentation/user-manual.txt @@ -3133,7 +3133,7 @@ those "loose" objects. You can save space and make Git faster by moving these loose objects in to a "pack file", which stores a group of objects in an efficient compressed format; the details of how pack files are formatted can be -found in link:technical/pack-format.html[pack format]. +found in link:gitformat-pack[5]. To put the loose objects into a pack, just run git repack: @@ -772,6 +772,7 @@ TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o +TEST_BUILTINS_OBJS += test-rot13-filter.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o TEST_BUILTINS_OBJS += test-run-command.o @@ -906,6 +907,7 @@ LIB_OBJS += blob.o LIB_OBJS += bloom.o LIB_OBJS += branch.o LIB_OBJS += bulk-checkin.o +LIB_OBJS += bundle-uri.o LIB_OBJS += bundle.o LIB_OBJS += cache-tree.o LIB_OBJS += cbtree.o @@ -918,6 +920,7 @@ LIB_OBJS += combine-diff.o LIB_OBJS += commit-graph.o LIB_OBJS += commit-reach.o LIB_OBJS += commit.o +LIB_OBJS += compat/nonblock.o LIB_OBJS += compat/obstack.o LIB_OBJS += compat/terminal.o LIB_OBJS += compat/zlib-uncompress2.o @@ -932,6 +935,7 @@ LIB_OBJS += ctype.o LIB_OBJS += date.o LIB_OBJS += decorate.o LIB_OBJS += delta-islands.o +LIB_OBJS += diagnose.o LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o @@ -992,7 +996,6 @@ LIB_OBJS += merge-ort.o LIB_OBJS += merge-ort-wrappers.o LIB_OBJS += merge-recursive.o LIB_OBJS += merge.o -LIB_OBJS += mergesort.o LIB_OBJS += midx.o LIB_OBJS += name-hash.o LIB_OBJS += negotiator/default.o @@ -1153,6 +1156,7 @@ BUILTIN_OBJS += builtin/credential-cache.o BUILTIN_OBJS += builtin/credential-store.o BUILTIN_OBJS += builtin/credential.o BUILTIN_OBJS += builtin/describe.o +BUILTIN_OBJS += builtin/diagnose.o BUILTIN_OBJS += builtin/diff-files.o BUILTIN_OBJS += builtin/diff-index.o BUILTIN_OBJS += builtin/diff-tree.o @@ -3532,6 +3536,7 @@ check-docs:: sed -e '1,/^### command list/d' \ -e '/^#/d' \ -e '/guide$$/d' \ + -e '/interfaces$$/d' \ -e 's/[ ].*//' \ -e 's/^/listed /' command-list.txt; \ $(MAKE) -C Documentation print-man1 | \ @@ -1023,7 +1023,7 @@ static int path_matches(const char *pathname, int pathlen, } return match_pathname(pathname, pathlen - isdir, base, baselen, - pattern, prefix, pat->patternlen, pat->flags); + pattern, prefix, pat->patternlen); } static int macroexpand_one(struct all_attrs_item *all_attrs, int nr, int rem); @@ -648,11 +648,14 @@ static struct commit_list *managed_skipped(struct commit_list *list, } static void bisect_rev_setup(struct repository *r, struct rev_info *revs, + struct strvec *rev_argv, const char *prefix, const char *bad_format, const char *good_format, int read_paths) { - struct strvec rev_argv = STRVEC_INIT; + struct setup_revision_opt opt = { + .free_removed_argv_elements = 1, + }; int i; repo_init_revisions(r, revs, prefix); @@ -660,17 +663,16 @@ static void bisect_rev_setup(struct repository *r, struct rev_info *revs, revs->commit_format = CMIT_FMT_UNSPECIFIED; /* rev_argv.argv[0] will be ignored by setup_revisions */ - strvec_push(&rev_argv, "bisect_rev_setup"); - strvec_pushf(&rev_argv, bad_format, oid_to_hex(current_bad_oid)); + strvec_push(rev_argv, "bisect_rev_setup"); + strvec_pushf(rev_argv, bad_format, oid_to_hex(current_bad_oid)); for (i = 0; i < good_revs.nr; i++) - strvec_pushf(&rev_argv, good_format, + strvec_pushf(rev_argv, good_format, oid_to_hex(good_revs.oid + i)); - strvec_push(&rev_argv, "--"); + strvec_push(rev_argv, "--"); if (read_paths) - read_bisect_paths(&rev_argv); + read_bisect_paths(rev_argv); - setup_revisions(rev_argv.nr, rev_argv.v, revs, NULL); - /* XXX leak rev_argv, as "revs" may still be pointing to it */ + setup_revisions(rev_argv->nr, rev_argv->v, revs, &opt); } static void bisect_common(struct rev_info *revs) @@ -873,10 +875,11 @@ static enum bisect_error check_merge_bases(int rev_nr, struct commit **rev, int static int check_ancestors(struct repository *r, int rev_nr, struct commit **rev, const char *prefix) { + struct strvec rev_argv = STRVEC_INIT; struct rev_info revs; int res; - bisect_rev_setup(r, &revs, prefix, "^%s", "%s", 0); + bisect_rev_setup(r, &revs, &rev_argv, prefix, "^%s", "%s", 0); bisect_common(&revs); res = (revs.commits != NULL); @@ -885,6 +888,7 @@ static int check_ancestors(struct repository *r, int rev_nr, clear_commit_marks_many(rev_nr, rev, ALL_REV_FLAGS); release_revisions(&revs); + strvec_clear(&rev_argv); return res; } @@ -1010,6 +1014,7 @@ void read_bisect_terms(const char **read_bad, const char **read_good) */ enum bisect_error bisect_next_all(struct repository *r, const char *prefix) { + struct strvec rev_argv = STRVEC_INIT; struct rev_info revs = REV_INFO_INIT; struct commit_list *tried; int reaches = 0, all = 0, nr, steps; @@ -1037,7 +1042,7 @@ enum bisect_error bisect_next_all(struct repository *r, const char *prefix) if (res) goto cleanup; - bisect_rev_setup(r, &revs, prefix, "%s", "^%s", 1); + bisect_rev_setup(r, &revs, &rev_argv, prefix, "%s", "^%s", 1); revs.first_parent_only = !!(bisect_flags & FIND_BISECTION_FIRST_PARENT_ONLY); revs.limited = 1; @@ -1054,7 +1059,7 @@ enum bisect_error bisect_next_all(struct repository *r, const char *prefix) */ res = error_if_skipped_commits(tried, NULL); if (res < 0) - return res; + goto cleanup; printf(_("%s was both %s and %s\n"), oid_to_hex(current_bad_oid), term_good, @@ -1112,6 +1117,7 @@ enum bisect_error bisect_next_all(struct repository *r, const char *prefix) res = bisect_checkout(bisect_rev, no_checkout); cleanup: release_revisions(&revs); + strvec_clear(&rev_argv); return res; } @@ -1098,30 +1098,22 @@ static struct blame_entry *blame_merge(struct blame_entry *list1, } } -static void *get_next_blame(const void *p) -{ - return ((struct blame_entry *)p)->next; -} - -static void set_next_blame(void *p1, void *p2) -{ - ((struct blame_entry *)p1)->next = p2; -} +DEFINE_LIST_SORT(static, sort_blame_entries, struct blame_entry, next); /* * Final image line numbers are all different, so we don't need a * three-way comparison here. */ -static int compare_blame_final(const void *p1, const void *p2) +static int compare_blame_final(const struct blame_entry *e1, + const struct blame_entry *e2) { - return ((struct blame_entry *)p1)->lno > ((struct blame_entry *)p2)->lno - ? 1 : -1; + return e1->lno > e2->lno ? 1 : -1; } -static int compare_blame_suspect(const void *p1, const void *p2) +static int compare_blame_suspect(const struct blame_entry *s1, + const struct blame_entry *s2) { - const struct blame_entry *s1 = p1, *s2 = p2; /* * to allow for collating suspects, we sort according to the * respective pointer value as the primary sorting criterion. @@ -1138,8 +1130,7 @@ static int compare_blame_suspect(const void *p1, const void *p2) void blame_sort_final(struct blame_scoreboard *sb) { - sb->ent = llist_mergesort(sb->ent, get_next_blame, set_next_blame, - compare_blame_final); + sort_blame_entries(&sb->ent, compare_blame_final); } static int compare_commits_by_reverse_commit_date(const void *a, @@ -1964,9 +1955,7 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, parent, target, 0); *d.dstq = NULL; if (ignore_diffs) - newdest = llist_mergesort(newdest, get_next_blame, - set_next_blame, - compare_blame_suspect); + sort_blame_entries(&newdest, compare_blame_suspect); queue_blames(sb, parent, newdest); return; @@ -2383,8 +2372,7 @@ static int num_scapegoats(struct rev_info *revs, struct commit *commit, int reve */ static void distribute_blame(struct blame_scoreboard *sb, struct blame_entry *blamed) { - blamed = llist_mergesort(blamed, get_next_blame, set_next_blame, - compare_blame_suspect); + sort_blame_entries(&blamed, compare_blame_suspect); while (blamed) { struct blame_origin *porigin = blamed->suspect; @@ -30,10 +30,9 @@ static inline unsigned char get_bitmask(uint32_t pos) static int load_bloom_filter_from_graph(struct commit_graph *g, struct bloom_filter *filter, - struct commit *c) + uint32_t graph_pos) { uint32_t lex_pos, start_index, end_index; - uint32_t graph_pos = commit_graph_position(c); while (graph_pos < g->num_commits_in_base) g = g->base_graph; @@ -203,9 +202,10 @@ struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, filter = bloom_filter_slab_at(&bloom_filters, c); if (!filter->data) { - load_commit_graph_info(r, c); - if (commit_graph_position(c) != COMMIT_NOT_FROM_GRAPH) - load_bloom_filter_from_graph(r->objects->commit_graph, filter, c); + uint32_t graph_pos; + if (repo_find_commit_pos_in_graph(r, c, &graph_pos)) + load_bloom_filter_from_graph(r->objects->commit_graph, + filter, graph_pos); } if (filter->data && filter->len) @@ -388,6 +388,7 @@ static void prepare_checked_out_branches(void) char *old; struct wt_status_state state = { 0 }; struct worktree *wt = worktrees[i++]; + struct string_list update_refs = STRING_LIST_INIT_DUP; if (wt->is_bare) continue; @@ -423,6 +424,18 @@ static void prepare_checked_out_branches(void) strbuf_release(&ref); } wt_status_state_free_buffers(&state); + + if (!sequencer_get_update_refs_state(get_worktree_git_dir(wt), + &update_refs)) { + struct string_list_item *item; + for_each_string_list_item(item, &update_refs) { + old = strmap_put(¤t_checked_out_branches, + item->string, + xstrdup(wt->path)); + free(old); + } + string_list_clear(&update_refs, 1); + } } free_worktrees(worktrees); @@ -144,6 +144,7 @@ int cmd_credential_cache(int argc, const char **argv, const char *prefix); int cmd_credential_cache_daemon(int argc, const char **argv, const char *prefix); int cmd_credential_store(int argc, const char **argv, const char *prefix); int cmd_describe(int argc, const char **argv, const char *prefix); +int cmd_diagnose(int argc, const char **argv, const char *prefix); int cmd_diff_files(int argc, const char **argv, const char *prefix); int cmd_diff_index(int argc, const char **argv, const char *prefix); int cmd_diff(int argc, const char **argv, const char *prefix); diff --git a/builtin/bugreport.c b/builtin/bugreport.c index 9de32bc96e..530895be55 100644 --- a/builtin/bugreport.c +++ b/builtin/bugreport.c @@ -5,6 +5,7 @@ #include "compat/compiler.h" #include "hook.h" #include "hook-list.h" +#include "diagnose.h" static void get_system_info(struct strbuf *sys_info) @@ -59,7 +60,7 @@ static void get_populated_hooks(struct strbuf *hook_info, int nongit) } static const char * const bugreport_usage[] = { - N_("git bugreport [-o|--output-directory <file>] [-s|--suffix <format>]"), + N_("git bugreport [-o|--output-directory <file>] [-s|--suffix <format>] [--diagnose[=<mode>]"), NULL }; @@ -98,16 +99,21 @@ int cmd_bugreport(int argc, const char **argv, const char *prefix) int report = -1; time_t now = time(NULL); struct tm tm; + enum diagnose_mode diagnose = DIAGNOSE_NONE; char *option_output = NULL; char *option_suffix = "%Y-%m-%d-%H%M"; const char *user_relative_path = NULL; char *prefixed_filename; + size_t output_path_len; const struct option bugreport_options[] = { + OPT_CALLBACK_F(0, "diagnose", &diagnose, N_("mode"), + N_("create an additional zip archive of detailed diagnostics (default 'stats')"), + PARSE_OPT_OPTARG, option_parse_diagnose), OPT_STRING('o', "output-directory", &option_output, N_("path"), - N_("specify a destination for the bugreport file")), + N_("specify a destination for the bugreport file(s)")), OPT_STRING('s', "suffix", &option_suffix, N_("format"), - N_("specify a strftime format suffix for the filename")), + N_("specify a strftime format suffix for the filename(s)")), OPT_END() }; @@ -119,6 +125,7 @@ int cmd_bugreport(int argc, const char **argv, const char *prefix) option_output ? option_output : ""); strbuf_addstr(&report_path, prefixed_filename); strbuf_complete(&report_path, '/'); + output_path_len = report_path.len; strbuf_addstr(&report_path, "git-bugreport-"); strbuf_addftime(&report_path, option_suffix, localtime_r(&now, &tm), 0, 0); @@ -133,6 +140,20 @@ int cmd_bugreport(int argc, const char **argv, const char *prefix) report_path.buf); } + /* Prepare diagnostics, if requested */ + if (diagnose != DIAGNOSE_NONE) { + struct strbuf zip_path = STRBUF_INIT; + strbuf_add(&zip_path, report_path.buf, output_path_len); + strbuf_addstr(&zip_path, "git-diagnostics-"); + strbuf_addftime(&zip_path, option_suffix, localtime_r(&now, &tm), 0, 0); + strbuf_addstr(&zip_path, ".zip"); + + if (create_diagnostics_archive(&zip_path, diagnose)) + die_errno(_("unable to create diagnostics archive %s"), zip_path.buf); + + strbuf_release(&zip_path); + } + /* Prepare the report contents */ get_bug_template(&buffer); diff --git a/builtin/cat-file.c b/builtin/cat-file.c index f42782e955..989eee0bb4 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -16,6 +16,7 @@ #include "packfile.h" #include "object-store.h" #include "promisor-remote.h" +#include "mailmap.h" enum batch_mode { BATCH_MODE_CONTENTS, @@ -31,11 +32,28 @@ struct batch_options { int all_objects; int unordered; int transform_mode; /* may be 'w' or 'c' for --filters or --textconv */ + int nul_terminated; const char *format; }; static const char *force_path; +static struct string_list mailmap = STRING_LIST_INIT_NODUP; +static int use_mailmap; + +static char *replace_idents_using_mailmap(char *, size_t *); + +static char *replace_idents_using_mailmap(char *object_buf, size_t *size) +{ + struct strbuf sb = STRBUF_INIT; + const char *headers[] = { "author ", "committer ", "tagger ", NULL }; + + strbuf_attach(&sb, object_buf, *size, *size + 1); + apply_mailmap_to_header(&sb, headers, &mailmap); + *size = sb.len; + return strbuf_detach(&sb, NULL); +} + static int filter_object(const char *path, unsigned mode, const struct object_id *oid, char **buf, unsigned long *size) @@ -160,6 +178,12 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name, if (!buf) die("Cannot read object %s", obj_name); + if (use_mailmap) { + size_t s = size; + buf = replace_idents_using_mailmap(buf, &s); + size = cast_size_t_to_ulong(s); + } + /* otherwise just spit out the data */ break; @@ -193,6 +217,12 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name, } buf = read_object_with_reference(the_repository, &oid, exp_type_id, &size, NULL); + + if (use_mailmap) { + size_t s = size; + buf = replace_idents_using_mailmap(buf, &s); + size = cast_size_t_to_ulong(s); + } break; } default: @@ -360,11 +390,18 @@ static void print_object_or_die(struct batch_options *opt, struct expand_data *d void *contents; contents = read_object_file(oid, &type, &size); + + if (use_mailmap) { + size_t s = size; + contents = replace_idents_using_mailmap(contents, &s); + size = cast_size_t_to_ulong(s); + } + if (!contents) die("object %s disappeared", oid_to_hex(oid)); if (type != data->type) die("object %s changed type!?", oid_to_hex(oid)); - if (data->info.sizep && size != data->size) + if (data->info.sizep && size != data->size && !use_mailmap) die("object %s changed size!?", oid_to_hex(oid)); batch_write(opt, contents, size); @@ -614,12 +651,20 @@ static void batch_objects_command(struct batch_options *opt, struct queued_cmd *queued_cmd = NULL; size_t alloc = 0, nr = 0; - while (!strbuf_getline(&input, stdin)) { - int i; + while (1) { + int i, ret; const struct parse_cmd *cmd = NULL; const char *p = NULL, *cmd_end; struct queued_cmd call = {0}; + if (opt->nul_terminated) + ret = strbuf_getline_nul(&input, stdin); + else + ret = strbuf_getline(&input, stdin); + + if (ret) + break; + if (!input.len) die(_("empty command in input")); if (isspace(*input.buf)) @@ -763,7 +808,16 @@ static int batch_objects(struct batch_options *opt) goto cleanup; } - while (strbuf_getline(&input, stdin) != EOF) { + while (1) { + int ret; + if (opt->nul_terminated) + ret = strbuf_getline_nul(&input, stdin); + else + ret = strbuf_getline(&input, stdin); + + if (ret == EOF) + break; + if (data.split_on_whitespace) { /* * Split at first whitespace, tying off the beginning @@ -856,6 +910,8 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix) OPT_CMDMODE('s', NULL, &opt, N_("show object size"), 's'), OPT_BOOL(0, "allow-unknown-type", &unknown_type, N_("allow -s and -t to work with broken/corrupt objects")), + OPT_BOOL(0, "use-mailmap", &use_mailmap, N_("use mail map file")), + OPT_ALIAS(0, "mailmap", "use-mailmap"), /* Batch mode */ OPT_GROUP(N_("Batch objects requested on stdin (or --batch-all-objects)")), OPT_CALLBACK_F(0, "batch", &batch, N_("format"), @@ -866,6 +922,7 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix) N_("like --batch, but don't emit <contents>"), PARSE_OPT_OPTARG | PARSE_OPT_NONEG, batch_option_callback), + OPT_BOOL('z', NULL, &batch.nul_terminated, N_("stdin is NUL-terminated")), OPT_CALLBACK_F(0, "batch-command", &batch, N_("format"), N_("read commands from stdin"), PARSE_OPT_OPTARG | PARSE_OPT_NONEG, @@ -898,6 +955,9 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix) opt_cw = (opt == 'c' || opt == 'w'); opt_epts = (opt == 'e' || opt == 'p' || opt == 't' || opt == 's'); + if (use_mailmap) + read_mailmap(&mailmap); + /* --batch-all-objects? */ if (opt == 'b') batch.all_objects = 1; @@ -921,6 +981,9 @@ int cmd_cat_file(int argc, const char **argv, const char *prefix) else if (batch.all_objects) usage_msg_optf(_("'%s' requires a batch mode"), usage, options, "--batch-all-objects"); + else if (batch.nul_terminated) + usage_msg_optf(_("'%s' requires a batch mode"), usage, options, + "-z"); /* Batch defaults */ if (batch.buffer_output < 0) diff --git a/builtin/checkout.c b/builtin/checkout.c index 1109f1301f..f9d63d80b9 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -417,7 +417,7 @@ static int checkout_worktree(const struct checkout_opts *opts, mem_pool_discard(&ce_mem_pool, should_validate_cache_entries()); remove_marked_cache_entries(&the_index, 1); remove_scheduled_dirs(); - errs |= finish_delayed_checkout(&state, &nr_checkouts, opts->show_progress); + errs |= finish_delayed_checkout(&state, opts->show_progress); if (opts->count_checkout_paths) { if (nr_unmerged) @@ -626,6 +626,7 @@ static void show_local_changes(struct object *head, repo_init_revisions(the_repository, &rev, NULL); rev.diffopt.flags = opts->flags; rev.diffopt.output_format |= DIFF_FORMAT_NAME_STATUS; + rev.diffopt.flags.recursive = 1; diff_setup_done(&rev.diffopt); add_pending_object(&rev, head, NULL); run_diff_index(&rev, 0); diff --git a/builtin/clone.c b/builtin/clone.c index c4ff4643ec..e21d42dfee 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -34,6 +34,7 @@ #include "list-objects-filter-options.h" #include "hook.h" #include "bundle.h" +#include "bundle-uri.h" /* * Overall FIXMEs: @@ -77,6 +78,7 @@ static int option_filter_submodules = -1; /* unspecified */ static int config_filter_submodules = -1; /* unspecified */ static struct string_list server_options = STRING_LIST_INIT_NODUP; static int option_remote_submodules; +static const char *bundle_uri; static int recurse_submodules_cb(const struct option *opt, const char *arg, int unset) @@ -160,6 +162,8 @@ static struct option builtin_clone_options[] = { N_("any cloned submodules will use their remote-tracking branch")), OPT_BOOL(0, "sparse", &option_sparse_checkout, N_("initialize sparse-checkout file to include only files at root")), + OPT_STRING(0, "bundle-uri", &bundle_uri, + N_("uri"), N_("a URI for downloading bundles before fetching from origin remote")), OPT_END() }; @@ -933,6 +937,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) option_no_checkout = 1; } + if (bundle_uri && deepen) + die(_("--bundle-uri is incompatible with --depth, --shallow-since, and --shallow-exclude")); + repo_name = argv[0]; path = get_repo_path(repo_name, &is_bundle); @@ -1232,6 +1239,18 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (transport->smart_options && !deepen && !filter_options.choice) transport->smart_options->check_self_contained_and_connected = 1; + /* + * Before fetching from the remote, download and install bundle + * data from the --bundle-uri option. + */ + if (bundle_uri) { + /* At this point, we need the_repository to match the cloned repo. */ + if (repo_init(the_repository, git_dir, work_tree)) + warning(_("failed to initialize the repo, skipping bundle URI")); + else if (fetch_bundle_uri(the_repository, bundle_uri)) + warning(_("failed to fetch objects from bundle URI '%s'"), + bundle_uri); + } strvec_push(&transport_ls_refs_options.ref_prefixes, "HEAD"); refspec_ref_prefixes(&remote->fetch, diff --git a/builtin/diagnose.c b/builtin/diagnose.c new file mode 100644 index 0000000000..cd260c2015 --- /dev/null +++ b/builtin/diagnose.c @@ -0,0 +1,61 @@ +#include "builtin.h" +#include "parse-options.h" +#include "diagnose.h" + +static const char * const diagnose_usage[] = { + N_("git diagnose [-o|--output-directory <path>] [-s|--suffix <format>] [--mode=<mode>]"), + NULL +}; + +int cmd_diagnose(int argc, const char **argv, const char *prefix) +{ + struct strbuf zip_path = STRBUF_INIT; + time_t now = time(NULL); + struct tm tm; + enum diagnose_mode mode = DIAGNOSE_STATS; + char *option_output = NULL; + char *option_suffix = "%Y-%m-%d-%H%M"; + char *prefixed_filename; + + const struct option diagnose_options[] = { + OPT_STRING('o', "output-directory", &option_output, N_("path"), + N_("specify a destination for the diagnostics archive")), + OPT_STRING('s', "suffix", &option_suffix, N_("format"), + N_("specify a strftime format suffix for the filename")), + OPT_CALLBACK_F(0, "mode", &mode, N_("(stats|all)"), + N_("specify the content of the diagnostic archive"), + PARSE_OPT_NONEG, option_parse_diagnose), + OPT_END() + }; + + argc = parse_options(argc, argv, prefix, diagnose_options, + diagnose_usage, 0); + + /* Prepare the path to put the result */ + prefixed_filename = prefix_filename(prefix, + option_output ? option_output : ""); + strbuf_addstr(&zip_path, prefixed_filename); + strbuf_complete(&zip_path, '/'); + + strbuf_addstr(&zip_path, "git-diagnostics-"); + strbuf_addftime(&zip_path, option_suffix, localtime_r(&now, &tm), 0, 0); + strbuf_addstr(&zip_path, ".zip"); + + switch (safe_create_leading_directories(zip_path.buf)) { + case SCLD_OK: + case SCLD_EXISTS: + break; + default: + die_errno(_("could not create leading directories for '%s'"), + zip_path.buf); + } + + /* Prepare diagnostics */ + if (create_diagnostics_archive(&zip_path, mode)) + die_errno(_("unable to create diagnostics archive %s"), + zip_path.buf); + + free(prefixed_filename); + strbuf_release(&zip_path); + return 0; +} diff --git a/builtin/fetch.c b/builtin/fetch.c index fc5cecb483..368a0f5329 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -490,7 +490,9 @@ static void filter_prefetch_refspec(struct refspec *rs) continue; if (!rs->items[i].dst || (rs->items[i].src && - !strncmp(rs->items[i].src, "refs/tags/", 10))) { + !strncmp(rs->items[i].src, + ref_namespace[NAMESPACE_TAGS].ref, + strlen(ref_namespace[NAMESPACE_TAGS].ref)))) { int j; free(rs->items[i].src); @@ -506,7 +508,7 @@ static void filter_prefetch_refspec(struct refspec *rs) } old_dst = rs->items[i].dst; - strbuf_addstr(&new_dst, "refs/prefetch/"); + strbuf_addstr(&new_dst, ref_namespace[NAMESPACE_PREFETCH].ref); /* * If old_dst starts with "refs/", then place diff --git a/builtin/gc.c b/builtin/gc.c index 84549888f5..eb567110e7 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -910,12 +910,6 @@ static int fetch_remote(struct remote *remote, void *cbdata) static int maintenance_task_prefetch(struct maintenance_run_opts *opts) { - git_config_set_multivar_gently("log.excludedecoration", - "refs/prefetch/", - "refs/prefetch/", - CONFIG_FLAGS_FIXED_VALUE | - CONFIG_FLAGS_MULTI_REPLACE); - if (for_each_remote(fetch_remote, opts)) { error(_("failed to prefetch remotes")); return 1; diff --git a/builtin/help.c b/builtin/help.c index 222f994f86..09ac4289f1 100644 --- a/builtin/help.c +++ b/builtin/help.c @@ -43,6 +43,8 @@ static enum help_action { HELP_ACTION_ALL = 1, HELP_ACTION_GUIDES, HELP_ACTION_CONFIG, + HELP_ACTION_USER_INTERFACES, + HELP_ACTION_DEVELOPER_INTERFACES, HELP_ACTION_CONFIG_FOR_COMPLETION, HELP_ACTION_CONFIG_SECTIONS_FOR_COMPLETION, } cmd_mode; @@ -69,6 +71,12 @@ static struct option builtin_help_options[] = { OPT_CMDMODE('g', "guides", &cmd_mode, N_("print list of useful guides"), HELP_ACTION_GUIDES), + OPT_CMDMODE(0, "user-interfaces", &cmd_mode, + N_("print list of user-facing repository, command and file interfaces"), + HELP_ACTION_USER_INTERFACES), + OPT_CMDMODE(0, "developer-interfaces", &cmd_mode, + N_("print list of file formats, protocols and other developer interfaces"), + HELP_ACTION_DEVELOPER_INTERFACES), OPT_CMDMODE('c', "config", &cmd_mode, N_("print all configuration variable names"), HELP_ACTION_CONFIG), OPT_CMDMODE_F(0, "config-for-completion", &cmd_mode, "", @@ -81,9 +89,11 @@ static struct option builtin_help_options[] = { static const char * const builtin_help_usage[] = { "git help [-a|--all] [--[no-]verbose]] [--[no-]external-commands] [--[no-]aliases]", - N_("git help [[-i|--info] [-m|--man] [-w|--web]] [<command>]"), + N_("git help [[-i|--info] [-m|--man] [-w|--web]] [<command>|<doc>]"), "git help [-g|--guides]", "git help [-c|--config]", + "git help [--user-interfaces]", + "git help [--developer-interfaces]", NULL }; @@ -654,6 +664,14 @@ int cmd_help(int argc, const char **argv, const char *prefix) opt_mode_usage(argc, "--config-for-completion", help_format); list_config_help(SHOW_CONFIG_VARS); return 0; + case HELP_ACTION_USER_INTERFACES: + opt_mode_usage(argc, "--user-interfaces", help_format); + list_user_interfaces_help(); + return 0; + case HELP_ACTION_DEVELOPER_INTERFACES: + opt_mode_usage(argc, "--developer-interfaces", help_format); + list_developer_interfaces_help(); + return 0; case HELP_ACTION_CONFIG_SECTIONS_FOR_COMPLETION: opt_mode_usage(argc, "--config-sections-for-completion", help_format); diff --git a/builtin/log.c b/builtin/log.c index fb84a0d399..56e2d95e86 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -101,6 +101,20 @@ static int parse_decoration_style(const char *value) return -1; } +static int use_default_decoration_filter = 1; +static struct string_list decorate_refs_exclude = STRING_LIST_INIT_NODUP; +static struct string_list decorate_refs_exclude_config = STRING_LIST_INIT_NODUP; +static struct string_list decorate_refs_include = STRING_LIST_INIT_NODUP; + +static int clear_decorations_callback(const struct option *opt, + const char *arg, int unset) +{ + string_list_clear(&decorate_refs_include, 0); + string_list_clear(&decorate_refs_exclude, 0); + use_default_decoration_filter = 0; + return 0; +} + static int decorate_callback(const struct option *opt, const char *arg, int unset) { if (unset) @@ -162,18 +176,61 @@ static void cmd_log_init_defaults(struct rev_info *rev) parse_date_format(default_date_mode, &rev->date_mode); } +static void set_default_decoration_filter(struct decoration_filter *decoration_filter) +{ + int i; + char *value = NULL; + struct string_list *include = decoration_filter->include_ref_pattern; + const struct string_list *config_exclude = + git_config_get_value_multi("log.excludeDecoration"); + + if (config_exclude) { + struct string_list_item *item; + for_each_string_list_item(item, config_exclude) + string_list_append(decoration_filter->exclude_ref_config_pattern, + item->string); + } + + /* + * By default, decorate_all is disabled. Enable it if + * log.initialDecorationSet=all. Don't ever disable it by config, + * since the command-line takes precedent. + */ + if (use_default_decoration_filter && + !git_config_get_string("log.initialdecorationset", &value) && + !strcmp("all", value)) + use_default_decoration_filter = 0; + free(value); + + if (!use_default_decoration_filter || + decoration_filter->exclude_ref_pattern->nr || + decoration_filter->include_ref_pattern->nr || + decoration_filter->exclude_ref_config_pattern->nr) + return; + + /* + * No command-line or config options were given, so + * populate with sensible defaults. + */ + for (i = 0; i < ARRAY_SIZE(ref_namespace); i++) { + if (!ref_namespace[i].decoration) + continue; + + string_list_append(include, ref_namespace[i].ref); + } +} + static void cmd_log_init_finish(int argc, const char **argv, const char *prefix, struct rev_info *rev, struct setup_revision_opt *opt) { struct userformat_want w; int quiet = 0, source = 0, mailmap; static struct line_opt_callback_data line_cb = {NULL, NULL, STRING_LIST_INIT_DUP}; - static struct string_list decorate_refs_exclude = STRING_LIST_INIT_NODUP; - static struct string_list decorate_refs_exclude_config = STRING_LIST_INIT_NODUP; - static struct string_list decorate_refs_include = STRING_LIST_INIT_NODUP; - struct decoration_filter decoration_filter = {&decorate_refs_include, - &decorate_refs_exclude, - &decorate_refs_exclude_config}; + struct decoration_filter decoration_filter = { + .exclude_ref_pattern = &decorate_refs_exclude, + .include_ref_pattern = &decorate_refs_include, + .exclude_ref_config_pattern = &decorate_refs_exclude_config, + }; static struct revision_sources revision_sources; const struct option builtin_log_options[] = { @@ -181,6 +238,10 @@ static void cmd_log_init_finish(int argc, const char **argv, const char *prefix, OPT_BOOL(0, "source", &source, N_("show source")), OPT_BOOL(0, "use-mailmap", &mailmap, N_("use mail map file")), OPT_ALIAS(0, "mailmap", "use-mailmap"), + OPT_CALLBACK_F(0, "clear-decorations", NULL, NULL, + N_("clear all previously-defined decoration filters"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, + clear_decorations_callback), OPT_STRING_LIST(0, "decorate-refs", &decorate_refs_include, N_("pattern"), N_("only decorate refs that match <pattern>")), OPT_STRING_LIST(0, "decorate-refs-exclude", &decorate_refs_exclude, @@ -265,16 +326,7 @@ static void cmd_log_init_finish(int argc, const char **argv, const char *prefix, } if (decoration_style || rev->simplify_by_decoration) { - const struct string_list *config_exclude = - repo_config_get_value_multi(the_repository, - "log.excludeDecoration"); - - if (config_exclude) { - struct string_list_item *item; - for_each_string_list_item(item, config_exclude) - string_list_append(&decorate_refs_exclude_config, - item->string); - } + set_default_decoration_filter(&decoration_filter); if (decoration_style) rev->show_decorations = 1; @@ -668,10 +720,10 @@ static void show_setup_revisions_tweak(struct rev_info *rev, int cmd_show(int argc, const char **argv, const char *prefix) { struct rev_info rev; - struct object_array_entry *objects; + unsigned int i; struct setup_revision_opt opt; struct pathspec match_all; - int i, count, ret = 0; + int ret = 0; init_log_defaults(); git_config(git_log_config, NULL); @@ -698,12 +750,10 @@ int cmd_show(int argc, const char **argv, const char *prefix) if (!rev.no_walk) return cmd_log_deinit(cmd_log_walk(&rev), &rev); - count = rev.pending.nr; - objects = rev.pending.objects; rev.diffopt.no_free = 1; - for (i = 0; i < count && !ret; i++) { - struct object *o = objects[i].item; - const char *name = objects[i].name; + for (i = 0; i < rev.pending.nr && !ret; i++) { + struct object *o = rev.pending.objects[i].item; + const char *name = rev.pending.objects[i].name; switch (o->type) { case OBJ_BLOB: ret = show_blob_object(&o->oid, &rev, name); @@ -726,7 +776,7 @@ int cmd_show(int argc, const char **argv, const char *prefix) if (!o) ret = error(_("could not read object %s"), oid_to_hex(oid)); - objects[i].item = o; + rev.pending.objects[i].item = o; i--; break; } @@ -743,11 +793,24 @@ int cmd_show(int argc, const char **argv, const char *prefix) rev.shown_one = 1; break; case OBJ_COMMIT: - rev.pending.nr = rev.pending.alloc = 0; - rev.pending.objects = NULL; + { + struct object_array old; + struct object_array blank = OBJECT_ARRAY_INIT; + + memcpy(&old, &rev.pending, sizeof(old)); + memcpy(&rev.pending, &blank, sizeof(rev.pending)); + add_object_array(o, name, &rev.pending); ret = cmd_log_walk_no_free(&rev); + + /* + * No need for + * object_array_clear(&pending). It was + * cleared already in prepare_revision_walk() + */ + memcpy(&rev.pending, &old, sizeof(rev.pending)); break; + } default: ret = error(_("unknown type: %d"), o->type); } diff --git a/builtin/ls-files.c b/builtin/ls-files.c index e791b65e7e..779dc18e59 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -11,6 +11,7 @@ #include "quote.h" #include "dir.h" #include "builtin.h" +#include "strbuf.h" #include "tree.h" #include "cache-tree.h" #include "parse-options.h" @@ -48,6 +49,7 @@ static char *ps_matched; static const char *with_tree; static int exc_given; static int exclude_args; +static const char *format; static const char *tag_cached = ""; static const char *tag_unmerged = ""; @@ -85,6 +87,16 @@ static void write_name(const char *name) stdout, line_terminator); } +static void write_name_to_buf(struct strbuf *sb, const char *name) +{ + const char *rel = relative_path(name, prefix_len ? prefix : NULL, sb); + + if (line_terminator) + quote_c_style(rel, sb, NULL, 0); + else + strbuf_addstr(sb, rel); +} + static const char *get_tag(const struct cache_entry *ce, const char *tag) { static char alttag[4]; @@ -222,6 +234,73 @@ static void show_submodule(struct repository *superproject, repo_clear(&subrepo); } +struct show_index_data { + const char *pathname; + struct index_state *istate; + const struct cache_entry *ce; +}; + +static size_t expand_show_index(struct strbuf *sb, const char *start, + void *context) +{ + struct show_index_data *data = context; + const char *end; + const char *p; + size_t len = strbuf_expand_literal_cb(sb, start, NULL); + struct stat st; + + if (len) + return len; + if (*start != '(') + die(_("bad ls-files format: element '%s' " + "does not start with '('"), start); + + end = strchr(start + 1, ')'); + if (!end) + die(_("bad ls-files format: element '%s'" + "does not end in ')'"), start); + + len = end - start + 1; + if (skip_prefix(start, "(objectmode)", &p)) + strbuf_addf(sb, "%06o", data->ce->ce_mode); + else if (skip_prefix(start, "(objectname)", &p)) + strbuf_add_unique_abbrev(sb, &data->ce->oid, abbrev); + else if (skip_prefix(start, "(stage)", &p)) + strbuf_addf(sb, "%d", ce_stage(data->ce)); + else if (skip_prefix(start, "(eolinfo:index)", &p)) + strbuf_addstr(sb, S_ISREG(data->ce->ce_mode) ? + get_cached_convert_stats_ascii(data->istate, + data->ce->name) : ""); + else if (skip_prefix(start, "(eolinfo:worktree)", &p)) + strbuf_addstr(sb, !lstat(data->pathname, &st) && + S_ISREG(st.st_mode) ? + get_wt_convert_stats_ascii(data->pathname) : ""); + else if (skip_prefix(start, "(eolattr)", &p)) + strbuf_addstr(sb, get_convert_attr_ascii(data->istate, + data->pathname)); + else if (skip_prefix(start, "(path)", &p)) + write_name_to_buf(sb, data->pathname); + else + die(_("bad ls-files format: %%%.*s"), (int)len, start); + + return len; +} + +static void show_ce_fmt(struct repository *repo, const struct cache_entry *ce, + const char *format, const char *fullname) { + struct show_index_data data = { + .pathname = fullname, + .istate = repo->index, + .ce = ce, + }; + struct strbuf sb = STRBUF_INIT; + + strbuf_expand(&sb, format, expand_show_index, &data); + strbuf_addch(&sb, line_terminator); + fwrite(sb.buf, sb.len, 1, stdout); + strbuf_release(&sb); +} + static void show_ce(struct repository *repo, struct dir_struct *dir, const struct cache_entry *ce, const char *fullname, const char *tag) @@ -236,6 +315,12 @@ static void show_ce(struct repository *repo, struct dir_struct *dir, max_prefix_len, ps_matched, S_ISDIR(ce->ce_mode) || S_ISGITLINK(ce->ce_mode))) { + if (format) { + show_ce_fmt(repo, ce, format, fullname); + print_debug(ce); + return; + } + tag = get_tag(ce, tag); if (!show_stage) { @@ -675,6 +760,9 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) N_("suppress duplicate entries")), OPT_BOOL(0, "sparse", &show_sparse_dirs, N_("show sparse directories in the presence of a sparse index")), + OPT_STRING_F(0, "format", &format, N_("format"), + N_("format to use for the output"), + PARSE_OPT_NONEG), OPT_END() }; int ret = 0; @@ -699,6 +787,13 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) for (i = 0; i < exclude_list.nr; i++) { add_pattern(exclude_list.items[i].string, "", 0, pl, --exclude_args); } + + if (format && (show_stage || show_others || show_killed || + show_resolve_undo || skipping_duplicates || show_eol || show_tag)) + usage_msg_opt(_("--format cannot be used with -s, -o, -k, -t, " + "--resolve-undo, --deduplicate, --eol"), + ls_files_usage, builtin_ls_files_options); + if (show_tag || show_valid_bit || show_fsmonitor_bit) { tag_cached = "H "; tag_unmerged = "M "; diff --git a/builtin/merge.c b/builtin/merge.c index 23170f2d2a..f7c92c0e64 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -313,8 +313,16 @@ static int save_state(struct object_id *stash) int len; struct child_process cp = CHILD_PROCESS_INIT; struct strbuf buffer = STRBUF_INIT; + struct lock_file lock_file = LOCK_INIT; + int fd; int rc = -1; + fd = repo_hold_locked_index(the_repository, &lock_file, 0); + refresh_cache(REFRESH_QUIET); + if (0 <= fd) + repo_update_index_if_able(the_repository, &lock_file); + rollback_lock_file(&lock_file); + strvec_pushl(&cp.args, "stash", "create", NULL); cp.out = -1; cp.git_cmd = 1; @@ -375,22 +383,26 @@ static void reset_hard(const struct object_id *oid, int verbose) static void restore_state(const struct object_id *head, const struct object_id *stash) { - const char *args[] = { "stash", "apply", NULL, NULL }; - - if (is_null_oid(stash)) - return; + struct strvec args = STRVEC_INIT; reset_hard(head, 1); - args[2] = oid_to_hex(stash); + if (is_null_oid(stash)) + goto refresh_cache; + + strvec_pushl(&args, "stash", "apply", "--index", "--quiet", NULL); + strvec_push(&args, oid_to_hex(stash)); /* * It is OK to ignore error here, for example when there was * nothing to restore. */ - run_command_v_opt(args, RUN_GIT_CMD); + run_command_v_opt(args.v, RUN_GIT_CMD); + strvec_clear(&args); - refresh_cache(REFRESH_QUIET); +refresh_cache: + if (discard_cache() < 0 || read_cache() < 0) + die(_("could not read index")); } /* This is called when no merge was necessary. */ @@ -754,8 +766,10 @@ static int try_merge_strategy(const char *strategy, struct commit_list *common, else clean = merge_recursive(&o, head, remoteheads->item, reversed, &result); - if (clean < 0) - exit(128); + if (clean < 0) { + rollback_lock_file(&lock); + return 2; + } if (write_locked_index(&the_index, &lock, COMMIT_LOCK | SKIP_IF_UNCHANGED)) die(_("unable to write %s"), get_index_file()); @@ -1599,6 +1613,21 @@ int cmd_merge(int argc, const char **argv, const char *prefix) */ refresh_cache(REFRESH_QUIET); if (allow_trivial && fast_forward != FF_ONLY) { + /* + * Must first ensure that index matches HEAD before + * attempting a trivial merge. + */ + struct tree *head_tree = get_commit_tree(head_commit); + struct strbuf sb = STRBUF_INIT; + + if (repo_index_has_changes(the_repository, head_tree, + &sb)) { + error(_("Your local changes to the following files would be overwritten by merge:\n %s"), + sb.buf); + strbuf_release(&sb); + return 2; + } + /* See if it is really trivial. */ git_committer_info(IDENT_STRICT); printf(_("Trying really trivial in-index merge...\n")); @@ -1655,12 +1684,12 @@ int cmd_merge(int argc, const char **argv, const char *prefix) * tree in the index -- this means that the index must be in * sync with the head commit. The strategies are responsible * to ensure this. + * + * Stash away the local changes so that we can try more than one + * and/or recover from merge strategies bailing while leaving the + * index and working tree polluted. */ - if (use_strategies_nr == 1 || - /* - * Stash away the local changes so that we can try more than one. - */ - save_state(&stash)) + if (save_state(&stash)) oidclr(&stash); for (i = 0; !merge_was_ok && i < use_strategies_nr; i++) { diff --git a/builtin/rebase.c b/builtin/rebase.c index 70aa7c842f..56e4214b44 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -102,6 +102,7 @@ struct rebase_options { int reschedule_failed_exec; int reapply_cherry_picks; int fork_point; + int update_refs; }; #define REBASE_OPTIONS_INIT { \ @@ -298,6 +299,7 @@ static int do_interactive_rebase(struct rebase_options *opts, unsigned flags) ret = complete_action(the_repository, &replay, flags, shortrevisions, opts->onto_name, opts->onto, &opts->orig_head, &commands, opts->autosquash, + opts->update_refs, &todo_list); } @@ -800,6 +802,11 @@ static int rebase_config(const char *var, const char *value, void *data) return 0; } + if (!strcmp(var, "rebase.updaterefs")) { + opts->update_refs = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "rebase.reschedulefailedexec")) { opts->reschedule_failed_exec = git_config_bool(var, value); return 0; @@ -1124,6 +1131,9 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) OPT_BOOL(0, "autosquash", &options.autosquash, N_("move commits that begin with " "squash!/fixup! under -i")), + OPT_BOOL(0, "update-refs", &options.update_refs, + N_("update branches that point to commits " + "that are being rebased")), { OPTION_STRING, 'S', "gpg-sign", &gpg_sign, N_("key-id"), N_("GPG-sign commits"), PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, diff --git a/builtin/reflog.c b/builtin/reflog.c index d3f6d903fb..461c94f1b2 100644 --- a/builtin/reflog.c +++ b/builtin/reflog.c @@ -193,6 +193,8 @@ static int expire_unreachable_callback(const struct option *opt, { struct cmd_reflog_expire_cb *cmd = opt->value; + BUG_ON_OPT_NEG(unset); + if (parse_expiry_date(arg, &cmd->expire_unreachable)) die(_("invalid timestamp '%s' given to '--%s'"), arg, opt->long_name); @@ -207,6 +209,8 @@ static int expire_total_callback(const struct option *opt, { struct cmd_reflog_expire_cb *cmd = opt->value; + BUG_ON_OPT_NEG(unset); + if (parse_expiry_date(arg, &cmd->expire_total)) die(_("invalid timestamp '%s' given to '--%s'"), arg, opt->long_name); diff --git a/builtin/remote.c b/builtin/remote.c index 9aff864fd6..272c7b8d9e 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -1231,10 +1231,9 @@ static int get_one_entry(struct remote *remote, void *priv) static int show_all(void) { - struct string_list list = STRING_LIST_INIT_NODUP; + struct string_list list = STRING_LIST_INIT_DUP; int result; - list.strdup_strings = 1; result = for_each_remote(get_one_entry, &list); if (!result) { diff --git a/builtin/replace.c b/builtin/replace.c index 583702a098..a29e911d30 100644 --- a/builtin/replace.c +++ b/builtin/replace.c @@ -106,6 +106,7 @@ static int for_each_replace_name(const char **argv, each_replace_name_fn fn) size_t base_len; int had_error = 0; struct object_id oid; + const char *git_replace_ref_base = ref_namespace[NAMESPACE_REPLACE].ref; strbuf_addstr(&ref, git_replace_ref_base); base_len = ref.len; @@ -147,6 +148,8 @@ static int check_ref_valid(struct object_id *object, struct strbuf *ref, int force) { + const char *git_replace_ref_base = ref_namespace[NAMESPACE_REPLACE].ref; + strbuf_reset(ref); strbuf_addf(ref, "%s%s", git_replace_ref_base, oid_to_hex(object)); if (check_refname_format(ref->buf, 0)) diff --git a/builtin/reset.c b/builtin/reset.c index 344fff8f3a..fdce6f8c85 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -174,88 +174,6 @@ static void update_index_from_diff(struct diff_queue_struct *q, } } -static int pathspec_needs_expanded_index(const struct pathspec *pathspec) -{ - unsigned int i, pos; - int res = 0; - char *skip_worktree_seen = NULL; - - /* - * When using a magic pathspec, assume for the sake of simplicity that - * the index needs to be expanded to match all matchable files. - */ - if (pathspec->magic) - return 1; - - for (i = 0; i < pathspec->nr; i++) { - struct pathspec_item item = pathspec->items[i]; - - /* - * If the pathspec item has a wildcard, the index should be expanded - * if the pathspec has the possibility of matching a subset of entries inside - * of a sparse directory (but not the entire directory). - * - * If the pathspec item is a literal path, the index only needs to be expanded - * if a) the pathspec isn't in the sparse checkout cone (to make sure we don't - * expand for in-cone files) and b) it doesn't match any sparse directories - * (since we can reset whole sparse directories without expanding them). - */ - if (item.nowildcard_len < item.len) { - /* - * Special case: if the pattern is a path inside the cone - * followed by only wildcards, the pattern cannot match - * partial sparse directories, so we know we don't need to - * expand the index. - * - * Examples: - * - in-cone/foo***: doesn't need expanded index - * - not-in-cone/bar*: may need expanded index - * - **.c: may need expanded index - */ - if (strspn(item.original + item.nowildcard_len, "*") == item.len - item.nowildcard_len && - path_in_cone_mode_sparse_checkout(item.original, &the_index)) - continue; - - for (pos = 0; pos < active_nr; pos++) { - struct cache_entry *ce = active_cache[pos]; - - if (!S_ISSPARSEDIR(ce->ce_mode)) - continue; - - /* - * If the pre-wildcard length is longer than the sparse - * directory name and the sparse directory is the first - * component of the pathspec, need to expand the index. - */ - if (item.nowildcard_len > ce_namelen(ce) && - !strncmp(item.original, ce->name, ce_namelen(ce))) { - res = 1; - break; - } - - /* - * If the pre-wildcard length is shorter than the sparse - * directory and the pathspec does not match the whole - * directory, need to expand the index. - */ - if (!strncmp(item.original, ce->name, item.nowildcard_len) && - wildmatch(item.original, ce->name, 0)) { - res = 1; - break; - } - } - } else if (!path_in_cone_mode_sparse_checkout(item.original, &the_index) && - !matches_skip_worktree(pathspec, i, &skip_worktree_seen)) - res = 1; - - if (res > 0) - break; - } - - free(skip_worktree_seen); - return res; -} - static int read_from_tree(const struct pathspec *pathspec, struct object_id *tree_oid, int intent_to_add) @@ -273,7 +191,7 @@ static int read_from_tree(const struct pathspec *pathspec, opt.change = diff_change; opt.add_remove = diff_addremove; - if (pathspec->nr && the_index.sparse_index && pathspec_needs_expanded_index(pathspec)) + if (pathspec->nr && pathspec_needs_expanded_index(&the_index, pathspec)) ensure_full_index(&the_index); if (do_diff_cache(tree_oid, &opt)) diff --git a/builtin/rev-list.c b/builtin/rev-list.c index 30fd8e83ea..fba6f5d51f 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -46,6 +46,7 @@ static const char rev_list_usage[] = " --parents\n" " --children\n" " --objects | --objects-edge\n" +" --disk-usage[=human]\n" " --unpacked\n" " --header | --pretty\n" " --[no-]object-names\n" @@ -81,6 +82,7 @@ static int arg_show_object_names = 1; static int show_disk_usage; static off_t total_disk_usage; +static int human_readable; static off_t get_object_disk_usage(struct object *obj) { @@ -368,6 +370,17 @@ static int show_object_fast( return 1; } +static void print_disk_usage(off_t size) +{ + struct strbuf sb = STRBUF_INIT; + if (human_readable) + strbuf_humanise_bytes(&sb, size); + else + strbuf_addf(&sb, "%"PRIuMAX, (uintmax_t)size); + puts(sb.buf); + strbuf_release(&sb); +} + static inline int parse_missing_action_value(const char *value) { if (!strcmp(value, "error")) { @@ -473,6 +486,7 @@ static int try_bitmap_disk_usage(struct rev_info *revs, int filter_provided_objects) { struct bitmap_index *bitmap_git; + off_t size_from_bitmap; if (!show_disk_usage) return -1; @@ -481,8 +495,8 @@ static int try_bitmap_disk_usage(struct rev_info *revs, if (!bitmap_git) return -1; - printf("%"PRIuMAX"\n", - (uintmax_t)get_disk_usage_from_bitmap(bitmap_git, revs)); + size_from_bitmap = get_disk_usage_from_bitmap(bitmap_git, revs); + print_disk_usage(size_from_bitmap); return 0; } @@ -624,7 +638,21 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix) continue; } - if (!strcmp(arg, "--disk-usage")) { + if (skip_prefix(arg, "--disk-usage", &arg)) { + if (*arg == '=') { + if (!strcmp(++arg, "human")) { + human_readable = 1; + } else + die(_("invalid value for '%s': '%s', the only allowed format is '%s'"), + "--disk-usage=<format>", arg, "human"); + } else if (*arg) { + /* + * Arguably should goto a label to continue chain of ifs? + * Doesn't matter unless we try to add --disk-usage-foo + * afterwards. + */ + usage(rev_list_usage); + } show_disk_usage = 1; info.flags |= REV_LIST_QUIET; continue; @@ -753,7 +781,7 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix) } if (show_disk_usage) - printf("%"PRIuMAX"\n", (uintmax_t)total_disk_usage); + print_disk_usage(total_disk_usage); cleanup: release_revisions(&revs); diff --git a/builtin/rm.c b/builtin/rm.c index 84a935a16e..b6ba859fe4 100644 --- a/builtin/rm.c +++ b/builtin/rm.c @@ -287,6 +287,8 @@ int cmd_rm(int argc, const char **argv, const char *prefix) if (!index_only) setup_work_tree(); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; hold_locked_index(&lock_file, LOCK_DIE_ON_ERROR); if (read_cache() < 0) @@ -296,8 +298,9 @@ int cmd_rm(int argc, const char **argv, const char *prefix) seen = xcalloc(pathspec.nr, 1); - /* TODO: audit for interaction with sparse-index. */ - ensure_full_index(&the_index); + if (pathspec_needs_expanded_index(&the_index, &pathspec)) + ensure_full_index(&the_index); + for (i = 0; i < active_nr; i++) { const struct cache_entry *ce = active_cache[i]; diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index fac52ade5e..b63f420ece 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1104,6 +1104,9 @@ static int compute_summary_module_list(struct object_id *head_oid, { struct strvec diff_args = STRVEC_INIT; struct rev_info rev; + struct setup_revision_opt opt = { + .free_removed_argv_elements = 1, + }; struct module_cb_list list = MODULE_CB_LIST_INIT; int ret = 0; @@ -1121,7 +1124,7 @@ static int compute_summary_module_list(struct object_id *head_oid, init_revisions(&rev, info->prefix); rev.abbrev = 0; precompose_argv_prefix(diff_args.nr, diff_args.v, NULL); - setup_revisions(diff_args.nr, diff_args.v, &rev, NULL); + setup_revisions(diff_args.nr, diff_args.v, &rev, &opt); rev.diffopt.output_format = DIFF_FORMAT_NO_OUTPUT | DIFF_FORMAT_CALLBACK; rev.diffopt.format_callback = submodule_summary_callback; rev.diffopt.format_callback_data = &list; diff --git a/builtin/symbolic-ref.c b/builtin/symbolic-ref.c index e547a08d6c..1b0f10225f 100644 --- a/builtin/symbolic-ref.c +++ b/builtin/symbolic-ref.c @@ -71,6 +71,8 @@ int cmd_symbolic_ref(int argc, const char **argv, const char *prefix) if (!strcmp(argv[0], "HEAD") && !starts_with(argv[1], "refs/")) die("Refusing to point HEAD outside of refs/"); + if (check_refname_format(argv[1], REFNAME_ALLOW_ONELEVEL) < 0) + die("Refusing to set '%s' to invalid ref '%s'", argv[0], argv[1]); ret = !!create_symref(argv[0], argv[1], msg); break; default: diff --git a/bulk-checkin.c b/bulk-checkin.c index 98ec893842..855b68ec23 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -340,6 +340,8 @@ void fsync_loose_object_bulk_checkin(int fd, const char *filename) */ if (!bulk_fsync_objdir || git_fsync(fd, FSYNC_WRITEOUT_ONLY) < 0) { + if (errno == ENOSYS) + warning(_("core.fsyncMethod = batch is unsupported on this platform")); fsync_or_die(fd, filename); } } diff --git a/bundle-uri.c b/bundle-uri.c new file mode 100644 index 0000000000..4a8cc74ed0 --- /dev/null +++ b/bundle-uri.c @@ -0,0 +1,168 @@ +#include "cache.h" +#include "bundle-uri.h" +#include "bundle.h" +#include "object-store.h" +#include "refs.h" +#include "run-command.h" + +static int find_temp_filename(struct strbuf *name) +{ + int fd; + /* + * Find a temporary filename that is available. This is briefly + * racy, but unlikely to collide. + */ + fd = odb_mkstemp(name, "bundles/tmp_uri_XXXXXX"); + if (fd < 0) { + warning(_("failed to create temporary file")); + return -1; + } + + close(fd); + unlink(name->buf); + return 0; +} + +static int download_https_uri_to_file(const char *file, const char *uri) +{ + int result = 0; + struct child_process cp = CHILD_PROCESS_INIT; + FILE *child_in = NULL, *child_out = NULL; + struct strbuf line = STRBUF_INIT; + int found_get = 0; + + strvec_pushl(&cp.args, "git-remote-https", uri, NULL); + cp.in = -1; + cp.out = -1; + + if (start_command(&cp)) + return 1; + + child_in = fdopen(cp.in, "w"); + if (!child_in) { + result = 1; + goto cleanup; + } + + child_out = fdopen(cp.out, "r"); + if (!child_out) { + result = 1; + goto cleanup; + } + + fprintf(child_in, "capabilities\n"); + fflush(child_in); + + while (!strbuf_getline(&line, child_out)) { + if (!line.len) + break; + if (!strcmp(line.buf, "get")) + found_get = 1; + } + strbuf_release(&line); + + if (!found_get) { + result = error(_("insufficient capabilities")); + goto cleanup; + } + + fprintf(child_in, "get %s %s\n\n", uri, file); + +cleanup: + if (child_in) + fclose(child_in); + if (finish_command(&cp)) + return 1; + if (child_out) + fclose(child_out); + return result; +} + +static int copy_uri_to_file(const char *filename, const char *uri) +{ + const char *out; + + if (starts_with(uri, "https:") || + starts_with(uri, "http:")) + return download_https_uri_to_file(filename, uri); + + if (skip_prefix(uri, "file://", &out)) + uri = out; + + /* Copy as a file */ + return copy_file(filename, uri, 0); +} + +static int unbundle_from_file(struct repository *r, const char *file) +{ + int result = 0; + int bundle_fd; + struct bundle_header header = BUNDLE_HEADER_INIT; + struct string_list_item *refname; + struct strbuf bundle_ref = STRBUF_INIT; + size_t bundle_prefix_len; + + if ((bundle_fd = read_bundle_header(file, &header)) < 0) + return 1; + + if ((result = unbundle(r, &header, bundle_fd, NULL))) + return 1; + + /* + * Convert all refs/heads/ from the bundle into refs/bundles/ + * in the local repository. + */ + strbuf_addstr(&bundle_ref, "refs/bundles/"); + bundle_prefix_len = bundle_ref.len; + + for_each_string_list_item(refname, &header.references) { + struct object_id *oid = refname->util; + struct object_id old_oid; + const char *branch_name; + int has_old; + + if (!skip_prefix(refname->string, "refs/heads/", &branch_name)) + continue; + + strbuf_setlen(&bundle_ref, bundle_prefix_len); + strbuf_addstr(&bundle_ref, branch_name); + + has_old = !read_ref(bundle_ref.buf, &old_oid); + update_ref("fetched bundle", bundle_ref.buf, oid, + has_old ? &old_oid : NULL, + REF_SKIP_OID_VERIFICATION, + UPDATE_REFS_MSG_ON_ERR); + } + + bundle_header_release(&header); + return result; +} + +int fetch_bundle_uri(struct repository *r, const char *uri) +{ + int result = 0; + struct strbuf filename = STRBUF_INIT; + + if ((result = find_temp_filename(&filename))) + goto cleanup; + + if ((result = copy_uri_to_file(filename.buf, uri))) { + warning(_("failed to download bundle from URI '%s'"), uri); + goto cleanup; + } + + if ((result = !is_bundle(filename.buf, 0))) { + warning(_("file at URI '%s' is not a bundle"), uri); + goto cleanup; + } + + if ((result = unbundle_from_file(r, filename.buf))) { + warning(_("failed to unbundle bundle from URI '%s'"), uri); + goto cleanup; + } + +cleanup: + unlink(filename.buf); + strbuf_release(&filename); + return result; +} diff --git a/bundle-uri.h b/bundle-uri.h new file mode 100644 index 0000000000..8a152f1ef1 --- /dev/null +++ b/bundle-uri.h @@ -0,0 +1,14 @@ +#ifndef BUNDLE_URI_H +#define BUNDLE_URI_H + +struct repository; + +/** + * Fetch data from the given 'uri' and unbundle the bundle data found + * based on that information. + * + * Returns non-zero if no bundle information is found at the given 'uri'. + */ +int fetch_bundle_uri(struct repository *r, const char *uri); + +#endif diff --git a/cache-tree.c b/cache-tree.c index 56db0b5026..c97111cccf 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -857,9 +857,7 @@ int cache_tree_matches_traversal(struct cache_tree *root, return 0; } -static void verify_one_sparse(struct repository *r, - struct index_state *istate, - struct cache_tree *it, +static void verify_one_sparse(struct index_state *istate, struct strbuf *path, int pos) { @@ -910,7 +908,7 @@ static int verify_one(struct repository *r, return 1; if (pos >= 0) { - verify_one_sparse(r, istate, it, path, pos); + verify_one_sparse(istate, path, pos); return 0; } @@ -475,8 +475,7 @@ extern struct index_state the_index; /* * Values in this enum (except those outside the 3 bit range) are part - * of pack file format. See Documentation/technical/pack-format.txt - * for more information. + * of pack file format. See gitformat-pack(5) for more information. */ enum object_type { OBJ_BAD = -1, @@ -831,6 +830,15 @@ struct cache_entry *index_file_exists(struct index_state *istate, const char *na int index_name_pos(struct index_state *, const char *name, int namelen); /* + * Like index_name_pos, returns the position of an entry of the given name in + * the index if one exists, otherwise returns a negative value where the negated + * value minus 1 is the position where the index entry would be inserted. Unlike + * index_name_pos, however, a sparse index is not expanded to find an entry + * inside a sparse directory. + */ +int index_name_pos_sparse(struct index_state *, const char *name, int namelen); + +/* * Determines whether an entry with the given name exists within the * given index. The return value is 1 if an exact match is found, otherwise * it is 0. Note that, unlike index_name_pos, this function does not expand @@ -1008,7 +1016,6 @@ void reset_shared_repository(void); * commands that do not want replace references to be active. */ extern int read_replace_refs; -extern char *git_replace_ref_base; /* * These values are used to help identify parts of a repository to fsync. @@ -1689,6 +1696,12 @@ struct ident_split { int split_ident_line(struct ident_split *, const char *, int); /* + * Given a commit or tag object buffer and the commit or tag headers, replaces + * the idents in the headers with their canonical versions using the mailmap mechanism. + */ +void apply_mailmap_to_header(struct strbuf *, const char **, struct string_list *); + +/* * Compare split idents for equality or strict ordering. Note that we * compare only the ident part of the line, ignoring any timestamp. * @@ -276,6 +276,7 @@ linux-musl) linux-leaks) export SANITIZE=leak export GIT_TEST_PASSING_SANITIZE_LEAK=true + export GIT_TEST_SANITIZE_LEAK_LOG=true ;; esac diff --git a/command-list.txt b/command-list.txt index 9bd6f3c48f..f96bdabd7d 100644 --- a/command-list.txt +++ b/command-list.txt @@ -43,6 +43,15 @@ # specified here, which can only have "guide" attribute and nothing # else. # +# User-facing repository, command and file interfaces such as +# documentation for the .gitmodules, .mailmap etc. files lives in man +# sections 5 and 7. These entries can only have the "userinterfaces" +# attribute and nothing else. +# +# Git's file formats and protocols, such as documentation for the +# *.bundle format lives in man section 5. These entries can only have +# the "developerinterfaces" attribute and nothing else. +# ### command list (do not change this line) # command name category [category] [category] git-add mainporcelain worktree @@ -192,24 +201,35 @@ git-verify-tag ancillaryinterrogators git-whatchanged ancillaryinterrogators complete git-worktree mainporcelain git-write-tree plumbingmanipulators -gitattributes guide -gitcli guide +gitattributes userinterfaces +gitcli userinterfaces gitcore-tutorial guide gitcredentials guide gitcvs-migration guide gitdiffcore guide giteveryday guide gitfaq guide +gitformat-bundle developerinterfaces +gitformat-chunk developerinterfaces +gitformat-commit-graph developerinterfaces +gitformat-index developerinterfaces +gitformat-pack developerinterfaces +gitformat-signature developerinterfaces gitglossary guide -githooks guide -gitignore guide +githooks userinterfaces +gitignore userinterfaces gitk mainporcelain -gitmailmap guide -gitmodules guide +gitmailmap userinterfaces +gitmodules userinterfaces gitnamespaces guide +gitprotocol-capabilities developerinterfaces +gitprotocol-common developerinterfaces +gitprotocol-http developerinterfaces +gitprotocol-pack developerinterfaces +gitprotocol-v2 developerinterfaces gitremote-helpers guide -gitrepository-layout guide -gitrevisions guide +gitrepository-layout userinterfaces +gitrevisions userinterfaces gitsubmodules guide gittutorial guide gittutorial-2 guide diff --git a/commit-graph.c b/commit-graph.c index 2b04ef072d..f2a36032f8 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -252,7 +252,8 @@ struct commit_graph *load_commit_graph_one_fd_st(struct repository *r, } graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - ret = parse_commit_graph(r, graph_map, graph_size); + prepare_repo_settings(r); + ret = parse_commit_graph(&r->settings, graph_map, graph_size); if (ret) ret->odb = odb; @@ -321,7 +322,7 @@ static int graph_read_bloom_data(const unsigned char *chunk_start, return 0; } -struct commit_graph *parse_commit_graph(struct repository *r, +struct commit_graph *parse_commit_graph(struct repo_settings *s, void *graph_map, size_t graph_size) { const unsigned char *data; @@ -359,8 +360,6 @@ struct commit_graph *parse_commit_graph(struct repository *r, return NULL; } - prepare_repo_settings(r); - graph = alloc_commit_graph(); graph->hash_len = the_hash_algo->rawsz; @@ -390,7 +389,7 @@ struct commit_graph *parse_commit_graph(struct repository *r, pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges); pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs); - if (get_configured_generation_version(r) >= 2) { + if (s->commit_graph_generation_version >= 2) { pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, &graph->chunk_generation_data); pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW, @@ -400,7 +399,7 @@ struct commit_graph *parse_commit_graph(struct repository *r, graph->read_generation_data = 1; } - if (r->settings.commit_graph_read_changed_paths) { + if (s->commit_graph_read_changed_paths) { pair_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, &graph->chunk_bloom_indexes); read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA, @@ -889,6 +888,14 @@ static int find_commit_pos_in_graph(struct commit *item, struct commit_graph *g, } } +int repo_find_commit_pos_in_graph(struct repository *r, struct commit *c, + uint32_t *pos) +{ + if (!prepare_commit_graph(r)) + return 0; + return find_commit_pos_in_graph(c, r->objects->commit_graph, pos); +} + struct commit *lookup_commit_in_graph(struct repository *repo, const struct object_id *id) { struct commit *commit; @@ -946,9 +953,7 @@ int parse_commit_in_graph(struct repository *r, struct commit *item) void load_commit_graph_info(struct repository *r, struct commit *item) { uint32_t pos; - if (!prepare_commit_graph(r)) - return; - if (find_commit_pos_in_graph(item, r->objects->commit_graph, &pos)) + if (repo_find_commit_pos_in_graph(r, item, &pos)) fill_commit_graph_info(item, r->objects->commit_graph, pos); } diff --git a/commit-graph.h b/commit-graph.h index 2e3ac35237..37faee6b66 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -41,6 +41,21 @@ int open_commit_graph(const char *graph_file, int *fd, struct stat *st); int parse_commit_in_graph(struct repository *r, struct commit *item); /* + * Fills `*pos` with the graph position of `c`, and returns 1 if `c` is + * found in the commit-graph belonging to `r`, or 0 otherwise. + * Initializes the commit-graph belonging to `r` if it hasn't been + * already. + * + * Note: this is a low-level helper that does not alter any slab data + * associated with `c`. Useful in circumstances where the slab data is + * already being modified (e.g., writing the commit-graph itself). + * + * In most cases, callers should use `parse_commit_in_graph()` instead. + */ +int repo_find_commit_pos_in_graph(struct repository *r, struct commit *c, + uint32_t *pos); + +/* * Look up the given commit ID in the commit-graph. This will only return a * commit if the ID exists both in the graph and in the object database such * that we don't return commits whose object has been pruned. Otherwise, this @@ -93,7 +108,12 @@ struct commit_graph *load_commit_graph_one_fd_st(struct repository *r, struct object_directory *odb); struct commit_graph *read_commit_graph_one(struct repository *r, struct object_directory *odb); -struct commit_graph *parse_commit_graph(struct repository *r, + +/* + * Callers should initialize the repo_settings with prepare_repo_settings() + * prior to calling parse_commit_graph(). + */ +struct commit_graph *parse_commit_graph(struct repo_settings *s, void *graph_map, size_t graph_size); /* @@ -642,10 +642,11 @@ struct commit_list * commit_list_insert_by_date(struct commit *item, struct comm return commit_list_insert(item, pp); } -static int commit_list_compare_by_date(const void *a, const void *b) +static int commit_list_compare_by_date(const struct commit_list *a, + const struct commit_list *b) { - timestamp_t a_date = ((const struct commit_list *)a)->item->date; - timestamp_t b_date = ((const struct commit_list *)b)->item->date; + timestamp_t a_date = a->item->date; + timestamp_t b_date = b->item->date; if (a_date < b_date) return 1; if (a_date > b_date) @@ -653,20 +654,11 @@ static int commit_list_compare_by_date(const void *a, const void *b) return 0; } -static void *commit_list_get_next(const void *a) -{ - return ((const struct commit_list *)a)->next; -} - -static void commit_list_set_next(void *a, void *next) -{ - ((struct commit_list *)a)->next = next; -} +DEFINE_LIST_SORT(static, commit_list_sort, struct commit_list, next); void commit_list_sort_by_date(struct commit_list **list) { - *list = llist_mergesort(*list, commit_list_get_next, commit_list_set_next, - commit_list_compare_by_date); + commit_list_sort(list, commit_list_compare_by_date); } struct commit *pop_most_recent_commit(struct commit_list **list, diff --git a/compat/disk.h b/compat/disk.h new file mode 100644 index 0000000000..50a32e3d8a --- /dev/null +++ b/compat/disk.h @@ -0,0 +1,56 @@ +#ifndef COMPAT_DISK_H +#define COMPAT_DISK_H + +#include "git-compat-util.h" + +static int get_disk_info(struct strbuf *out) +{ + struct strbuf buf = STRBUF_INIT; + int res = 0; + +#ifdef GIT_WINDOWS_NATIVE + char volume_name[MAX_PATH], fs_name[MAX_PATH]; + DWORD serial_number, component_length, flags; + ULARGE_INTEGER avail2caller, total, avail; + + strbuf_realpath(&buf, ".", 1); + if (!GetDiskFreeSpaceExA(buf.buf, &avail2caller, &total, &avail)) { + error(_("could not determine free disk size for '%s'"), + buf.buf); + res = -1; + goto cleanup; + } + + strbuf_setlen(&buf, offset_1st_component(buf.buf)); + if (!GetVolumeInformationA(buf.buf, volume_name, sizeof(volume_name), + &serial_number, &component_length, &flags, + fs_name, sizeof(fs_name))) { + error(_("could not get info for '%s'"), buf.buf); + res = -1; + goto cleanup; + } + strbuf_addf(out, "Available space on '%s': ", buf.buf); + strbuf_humanise_bytes(out, avail2caller.QuadPart); + strbuf_addch(out, '\n'); +#else + struct statvfs stat; + + strbuf_realpath(&buf, ".", 1); + if (statvfs(buf.buf, &stat) < 0) { + error_errno(_("could not determine free disk size for '%s'"), + buf.buf); + res = -1; + goto cleanup; + } + + strbuf_addf(out, "Available space on '%s': ", buf.buf); + strbuf_humanise_bytes(out, (off_t)stat.f_bsize * (off_t)stat.f_bavail); + strbuf_addf(out, " (mount flags 0x%lx)\n", stat.f_flag); +#endif + +cleanup: + strbuf_release(&buf); + return res; +} + +#endif /* COMPAT_DISK_H */ diff --git a/compat/mingw.c b/compat/mingw.c index b5502997e2..901375d584 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1,6 +1,7 @@ #include "../git-compat-util.h" #include "win32.h" #include <aclapi.h> +#include <sddl.h> #include <conio.h> #include <wchar.h> #include "../strbuf.h" @@ -768,8 +769,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) wfilename[n] = L'\0'; attributes = GetFileAttributesW(wfilename); wfilename[n] = c; - if (attributes == FILE_ATTRIBUTE_DIRECTORY || - attributes == FILE_ATTRIBUTE_DEVICE) + if (attributes & + (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_DEVICE)) return 1; if (attributes == INVALID_FILE_ATTRIBUTES) switch (GetLastError()) { @@ -2670,7 +2671,22 @@ static PSID get_current_user_sid(void) return result; } -int is_path_owned_by_current_sid(const char *path) +static int acls_supported(const char *path) +{ + size_t offset = offset_1st_component(path); + WCHAR wroot[MAX_PATH]; + DWORD file_system_flags; + + if (offset && + xutftowcsn(wroot, path, MAX_PATH, offset) > 0 && + GetVolumeInformationW(wroot, NULL, 0, NULL, NULL, + &file_system_flags, NULL, 0)) + return !!(file_system_flags & FILE_PERSISTENT_ACLS); + + return 0; +} + +int is_path_owned_by_current_sid(const char *path, struct strbuf *report) { WCHAR wpath[MAX_PATH]; PSID sid = NULL; @@ -2709,6 +2725,7 @@ int is_path_owned_by_current_sid(const char *path) else if (sid && IsValidSid(sid)) { /* Now, verify that the SID matches the current user's */ static PSID current_user_sid; + BOOL is_member; if (!current_user_sid) current_user_sid = get_current_user_sid(); @@ -2717,6 +2734,46 @@ int is_path_owned_by_current_sid(const char *path) IsValidSid(current_user_sid) && EqualSid(sid, current_user_sid)) result = 1; + else if (IsWellKnownSid(sid, WinBuiltinAdministratorsSid) && + CheckTokenMembership(NULL, sid, &is_member) && + is_member) + /* + * If owned by the Administrators group, and the + * current user is an administrator, we consider that + * okay, too. + */ + result = 1; + else if (report && + IsWellKnownSid(sid, WinWorldSid) && + !acls_supported(path)) { + /* + * On FAT32 volumes, ownership is not actually recorded. + */ + strbuf_addf(report, "'%s' is on a file system that does" + "not record ownership\n", path); + } else if (report) { + LPSTR str1, str2, to_free1 = NULL, to_free2 = NULL; + + if (ConvertSidToStringSidA(sid, &str1)) + to_free1 = str1; + else + str1 = "(inconvertible)"; + + if (!current_user_sid) + str2 = "(none)"; + else if (!IsValidSid(current_user_sid)) + str2 = "(invalid)"; + else if (ConvertSidToStringSidA(current_user_sid, &str2)) + to_free2 = str2; + else + str2 = "(inconvertible)"; + strbuf_addf(report, + "'%s' is owned by:\n" + "\t'%s'\nbut the current user is:\n" + "\t'%s'\n", path, str1, str2); + LocalFree(to_free1); + LocalFree(to_free2); + } } /* diff --git a/compat/mingw.h b/compat/mingw.h index a74da68f31..209cf7ceba 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -463,7 +463,7 @@ char *mingw_query_user_email(void); * Verifies that the specified path is owned by the user running the * current process. */ -int is_path_owned_by_current_sid(const char *path); +int is_path_owned_by_current_sid(const char *path, struct strbuf *report); #define is_path_owned_by_current_user is_path_owned_by_current_sid /** diff --git a/compat/nonblock.c b/compat/nonblock.c new file mode 100644 index 0000000000..9694ebdb1d --- /dev/null +++ b/compat/nonblock.c @@ -0,0 +1,50 @@ +#include "git-compat-util.h" +#include "nonblock.h" + +#ifdef O_NONBLOCK + +int enable_pipe_nonblock(int fd) +{ + int flags = fcntl(fd, F_GETFL); + if (flags < 0) + return -1; + flags |= O_NONBLOCK; + return fcntl(fd, F_SETFL, flags); +} + +#elif defined(GIT_WINDOWS_NATIVE) + +#include "win32.h" + +int enable_pipe_nonblock(int fd) +{ + HANDLE h = (HANDLE)_get_osfhandle(fd); + DWORD mode; + DWORD type = GetFileType(h); + if (type == FILE_TYPE_UNKNOWN && GetLastError() != NO_ERROR) { + errno = EBADF; + return -1; + } + if (type != FILE_TYPE_PIPE) + BUG("unsupported file type: %lu", type); + if (!GetNamedPipeHandleState(h, &mode, NULL, NULL, NULL, NULL, 0)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + mode |= PIPE_NOWAIT; + if (!SetNamedPipeHandleState(h, &mode, NULL, NULL)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + return 0; +} + +#else + +int enable_pipe_nonblock(int fd) +{ + errno = ENOSYS; + return -1; +} + +#endif diff --git a/compat/nonblock.h b/compat/nonblock.h new file mode 100644 index 0000000000..af1a331301 --- /dev/null +++ b/compat/nonblock.h @@ -0,0 +1,9 @@ +#ifndef COMPAT_NONBLOCK_H +#define COMPAT_NONBLOCK_H + +/* + * Enable non-blocking I/O for the pipe specified by the passed-in descriptor. + */ +int enable_pipe_nonblock(int fd); + +#endif diff --git a/compat/win32/syslog.c b/compat/win32/syslog.c index 1f8d8934cc..0af18d8882 100644 --- a/compat/win32/syslog.c +++ b/compat/win32/syslog.c @@ -44,6 +44,7 @@ void syslog(int priority, const char *fmt, ...) while ((pos = strstr(str, "%1")) != NULL) { size_t offset = pos - str; + char *new_pos; char *oldstr = str; str = realloc(str, st_add(++str_len, 1)); if (!str) { @@ -51,9 +52,9 @@ void syslog(int priority, const char *fmt, ...) warning_errno("realloc failed"); return; } - pos = str + offset; - memmove(pos + 2, pos + 1, strlen(pos)); - pos[1] = ' '; + new_pos = str + offset; + memmove(new_pos + 2, new_pos + 1, strlen(new_pos)); + new_pos[1] = ' '; } switch (priority) { @@ -1979,6 +1979,8 @@ int git_config_from_file_with_options(config_fn_t fn, const char *filename, int ret = -1; FILE *f; + if (!filename) + BUG("filename cannot be NULL"); f = fopen_or_warn(filename, "r"); if (f) { ret = do_config_from_file(fn, CONFIG_ORIGIN_FILE, filename, @@ -2645,9 +2647,12 @@ static void read_protected_config(void) system_config = git_system_config(); git_global_config(&user_config, &xdg_config); - git_configset_add_file(&protected_config, system_config); - git_configset_add_file(&protected_config, xdg_config); - git_configset_add_file(&protected_config, user_config); + if (system_config) + git_configset_add_file(&protected_config, system_config); + if (xdg_config) + git_configset_add_file(&protected_config, xdg_config); + if (user_config) + git_configset_add_file(&protected_config, user_config); git_configset_add_parameters(&protected_config); free(system_config); diff --git a/config.mak.dev b/config.mak.dev index 335efd4620..4fa19d361b 100644 --- a/config.mak.dev +++ b/config.mak.dev @@ -59,9 +59,13 @@ endif # uninitialized warnings on gcc 4.9.2 in xdiff/xdiffi.c and config.c # not worth fixing since newer compilers correctly stop complaining +# +# Likewise, gcc older than 4.9 complains about initializing a +# struct-within-a-struct using just "{ 0 }" ifneq ($(filter gcc4,$(COMPILER_FEATURES)),) ifeq ($(filter gcc5,$(COMPILER_FEATURES)),) DEVELOPER_CFLAGS += -Wno-uninitialized +DEVELOPER_CFLAGS += -Wno-missing-braces endif endif diff --git a/config.mak.uname b/config.mak.uname index ce83cad47a..d63629fe80 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -656,7 +656,6 @@ ifeq ($(uname_S),MINGW) UNRELIABLE_FSTAT = UnfortunatelyYes OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo NO_REGEX = YesPlease - NO_PYTHON = YesPlease ETAGS_TARGET = ETAGS NO_POSIX_GOODIES = UnfortunatelyYes DEFAULT_HELP_FORMAT = html @@ -686,6 +685,7 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = YesPlease + NO_PYTHON = YesPlease COMPAT_CFLAGS += -D__USE_MINGW_ACCESS else ifneq ($(shell expr "$(uname_R)" : '1\.'),2) @@ -717,10 +717,8 @@ else INSTALL = /bin/install INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease - NO_GETTEXT = USE_GETTEXT_SCHEME = fallthrough USE_LIBPCRE = YesPlease - NO_CURL = USE_NED_ALLOCATOR = YesPlease ifeq (/mingw64,$(subst 32,64,$(prefix))) # Move system config into top-level /etc/ @@ -730,6 +728,7 @@ else else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease + NO_PYTHON = YesPlease endif endif endif diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 1b23f2440d..2237109b57 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -77,7 +77,7 @@ if(USE_VCPKG) set(CMAKE_TOOLCHAIN_FILE ${VCPKG_DIR}/scripts/buildsystems/vcpkg.cmake CACHE STRING "Vcpkg toolchain file") endif() -find_program(SH_EXE sh PATHS "C:/Program Files/Git/bin") +find_program(SH_EXE sh PATHS "C:/Program Files/Git/bin" "$ENV{LOCALAPPDATA}/Programs/Git/bin") if(NOT SH_EXE) message(FATAL_ERROR "sh: shell interpreter was not found in your path, please install one." "On Windows, you can get it as part of 'Git for Windows' install at https://gitforwindows.org/") diff --git a/contrib/completion/git-prompt.sh b/contrib/completion/git-prompt.sh index 1435548e00..57972c2845 100644 --- a/contrib/completion/git-prompt.sh +++ b/contrib/completion/git-prompt.sh @@ -84,6 +84,10 @@ # single '?' character by setting GIT_PS1_COMPRESSSPARSESTATE, or omitted # by setting GIT_PS1_OMITSPARSESTATE. # +# If you would like to see a notification on the prompt when there are +# unresolved conflicts, set GIT_PS1_SHOWCONFLICTSTATE to "yes". The +# prompt will include "|CONFLICT". +# # If you would like to see more information about the identity of # commits checked out as a detached HEAD, set GIT_PS1_DESCRIBE_STYLE # to one of these values: @@ -508,6 +512,12 @@ __git_ps1 () r="$r $step/$total" fi + local conflict="" # state indicator for unresolved conflicts + if [[ "${GIT_PS1_SHOWCONFLICTSTATE}" == "yes" ]] && + [[ $(git ls-files --unmerged 2>/dev/null) ]]; then + conflict="|CONFLICT" + fi + local w="" local i="" local s="" @@ -572,7 +582,7 @@ __git_ps1 () fi local f="$h$w$i$s$u$p" - local gitstring="$c$b${f:+$z$f}${sparse}$r${upstream}" + local gitstring="$c$b${f:+$z$f}${sparse}$r${upstream}${conflict}" if [ $pcmode = yes ]; then if [ "${__git_printf_supports_v-}" != yes ]; then diff --git a/contrib/credential/netrc/t-git-credential-netrc.sh b/contrib/credential/netrc/t-git-credential-netrc.sh index 07227d0228..bf2777308a 100755 --- a/contrib/credential/netrc/t-git-credential-netrc.sh +++ b/contrib/credential/netrc/t-git-credential-netrc.sh @@ -3,16 +3,9 @@ cd ../../../t test_description='git-credential-netrc' . ./test-lib.sh + . "$TEST_DIRECTORY"/lib-perl.sh - if ! test_have_prereq PERL; then - skip_all='skipping perl interface tests, perl not available' - test_done - fi - - perl -MTest::More -e 0 2>/dev/null || { - skip_all="Perl Test::More unavailable, skipping test" - test_done - } + skip_all_if_no_Test_More # set up test repository @@ -20,13 +13,10 @@ 'set up test repository' \ 'git config --add gpg.program test.git-config-gpg' - # The external test will outputs its own plan - test_external_has_tap=1 - export PERL5LIB="$GITPERLLIB" - test_external \ - 'git-credential-netrc' \ + test_expect_success 'git-credential-netrc' ' perl "$GIT_BUILD_DIR"/contrib/credential/netrc/test.pl + ' test_done ) diff --git a/contrib/credential/osxkeychain/git-credential-osxkeychain.c b/contrib/credential/osxkeychain/git-credential-osxkeychain.c index 0b44a9b7cc..bf77748d60 100644 --- a/contrib/credential/osxkeychain/git-credential-osxkeychain.c +++ b/contrib/credential/osxkeychain/git-credential-osxkeychain.c @@ -168,7 +168,7 @@ int main(int argc, const char **argv) "usage: git credential-osxkeychain <get|store|erase>"; if (!argv[1]) - die(usage); + die("%s", usage); read_credential(); diff --git a/contrib/rerere-train.sh b/contrib/rerere-train.sh index 26b724c8c6..bd01e430ef 100755 --- a/contrib/rerere-train.sh +++ b/contrib/rerere-train.sh @@ -75,7 +75,7 @@ do continue fi git checkout -q "$parent1^0" - if git merge $other_parents >/dev/null 2>&1 + if git merge --no-gpg-sign $other_parents >/dev/null 2>&1 then # Cleanly merges continue diff --git a/contrib/scalar/README.md b/contrib/scalar/README.md deleted file mode 100644 index 634b5771ed..0000000000 --- a/contrib/scalar/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Scalar - an opinionated repository management tool - -Scalar is an add-on to Git that helps users take advantage of advanced -performance features in Git. Originally implemented in C# using .NET Core, -based on the learnings from the VFS for Git project, most of the techniques -developed by the Scalar project have been integrated into core Git already: - -* partial clone, -* commit graphs, -* multi-pack index, -* sparse checkout (cone mode), -* scheduled background maintenance, -* etc - -This directory contains the remaining parts of Scalar that are not (yet) in -core Git. - -## Roadmap - -The idea is to populate this directory via incremental patch series and -eventually move to a top-level directory next to `gitk-git/` and to `git-gui/`. The -current plan involves the following patch series: - -- `scalar-the-beginning`: The initial patch series which sets up - `contrib/scalar/` and populates it with a minimal `scalar` command that - demonstrates the fundamental ideas. - -- `scalar-c-and-C`: The `scalar` command learns about two options that can be - specified before the command, `-c <key>=<value>` and `-C <directory>`. - -- `scalar-diagnose`: The `scalar` command is taught the `diagnose` subcommand. - -- `scalar-and-builtin-fsmonitor`: The built-in FSMonitor is enabled in `scalar - register` and in `scalar clone`, for an enormous performance boost when - working in large worktrees. This patch series necessarily depends on Jeff - Hostetler's FSMonitor patch series to be integrated into Git. - -- `scalar-gentler-config-locking`: Scalar enlistments are registered in the - user's Git config. This usually does not represent any problem because it is - rare for a user to register an enlistment. However, in Scalar's functional - tests, Scalar enlistments are created galore, and in parallel, which can lead - to lock contention. This patch series works around that problem by re-trying - to lock the config file in a gentle fashion. - -- `scalar-extra-docs`: Add some extensive documentation that has been written - in the original Scalar project (all subject to discussion, of course). - -- `optionally-install-scalar`: Now that Scalar is feature (and documentation) - complete and is verified in CI builds, let's offer to install it. - -- `move-scalar-to-toplevel`: Now that Scalar is complete, let's move it next to - `gitk-git/` and to `git-gui/`, making it a top-level command. - -The following two patch series exist in Microsoft's fork of Git and are -publicly available. There is no current plan to upstream them, not because I -want to withhold these patches, but because I don't think the Git community is -interested in these patches. - -There are some interesting ideas there, but the implementation is too specific -to Azure Repos and/or VFS for Git to be of much help in general (and also: my -colleagues tried to upstream some patches already and the enthusiasm for -integrating things related to Azure Repos and VFS for Git can be summarized in -very, very few words). - -These still exist mainly because the GVFS protocol is what Azure Repos has -instead of partial clone, while Git is focused on improving partial clone: - -- `scalar-with-gvfs`: The primary purpose of this patch series is to support - existing Scalar users whose repositories are hosted in Azure Repos (which - does not support Git's partial clones, but supports its predecessor, the GVFS - protocol, which is used by Scalar to emulate the partial clone). - - Since the GVFS protocol will never be supported by core Git, this patch - series will remain in Microsoft's fork of Git. - -- `run-scalar-functional-tests`: The Scalar project developed a quite - comprehensive set of integration tests (or, "Functional Tests"). They are the - sole remaining part of the original C#-based Scalar project, and this patch - adds a GitHub workflow that runs them all. - - Since the tests partially depend on features that are only provided in the - `scalar-with-gvfs` patch series, this patch cannot be upstreamed. diff --git a/contrib/scalar/scalar.c b/contrib/scalar/scalar.c index 97e71fe19c..642d16124e 100644 --- a/contrib/scalar/scalar.c +++ b/contrib/scalar/scalar.c @@ -7,27 +7,13 @@ #include "parse-options.h" #include "config.h" #include "run-command.h" +#include "simple-ipc.h" +#include "fsmonitor-ipc.h" +#include "fsmonitor-settings.h" #include "refs.h" #include "dir.h" #include "packfile.h" #include "help.h" -#include "archive.h" -#include "object-store.h" - -/* - * Remove the deepest subdirectory in the provided path string. Path must not - * include a trailing path separator. Returns 1 if parent directory found, - * otherwise 0. - */ -static int strbuf_parent_directory(struct strbuf *buf) -{ - size_t len = buf->len; - size_t offset = offset_1st_component(buf->buf); - char *path_sep = find_last_dir_sep(buf->buf + offset); - strbuf_setlen(buf, path_sep ? path_sep - buf->buf : offset); - - return buf->len < len; -} static void setup_enlistment_directory(int argc, const char **argv, const char * const *usagestr, @@ -35,8 +21,8 @@ static void setup_enlistment_directory(int argc, const char **argv, struct strbuf *enlistment_root) { struct strbuf path = STRBUF_INIT; - char *root; - int enlistment_found = 0; + int enlistment_is_repo_parent = 0; + size_t len; if (startup_info->have_repository) BUG("gitdir already set up?!?"); @@ -49,51 +35,36 @@ static void setup_enlistment_directory(int argc, const char **argv, strbuf_add_absolute_path(&path, argv[0]); if (!is_directory(path.buf)) die(_("'%s' does not exist"), path.buf); + if (chdir(path.buf) < 0) + die_errno(_("could not switch to '%s'"), path.buf); } else if (strbuf_getcwd(&path) < 0) die(_("need a working directory")); strbuf_trim_trailing_dir_sep(&path); - do { - const size_t len = path.len; - - /* check if currently in enlistment root with src/ workdir */ - strbuf_addstr(&path, "/src"); - if (is_nonbare_repository_dir(&path)) { - if (enlistment_root) - strbuf_add(enlistment_root, path.buf, len); - enlistment_found = 1; - break; - } - - /* reset to original path */ - strbuf_setlen(&path, len); - - /* check if currently in workdir */ - if (is_nonbare_repository_dir(&path)) { - if (enlistment_root) { - /* - * If the worktree's directory's name is `src`, the enlistment is the - * parent directory, otherwise it is identical to the worktree. - */ - root = strip_path_suffix(path.buf, "src"); - strbuf_addstr(enlistment_root, root ? root : path.buf); - free(root); - } + /* check if currently in enlistment root with src/ workdir */ + len = path.len; + strbuf_addstr(&path, "/src"); + if (is_nonbare_repository_dir(&path)) { + enlistment_is_repo_parent = 1; + if (chdir(path.buf) < 0) + die_errno(_("could not switch to '%s'"), path.buf); + } + strbuf_setlen(&path, len); - enlistment_found = 1; - break; - } - } while (strbuf_parent_directory(&path)); + setup_git_directory(); - if (!enlistment_found) - die(_("could not find enlistment root")); + if (!the_repository->worktree) + die(_("Scalar enlistments require a worktree")); - if (chdir(path.buf) < 0) - die_errno(_("could not switch to '%s'"), path.buf); + if (enlistment_root) { + if (enlistment_is_repo_parent) + strbuf_addbuf(enlistment_root, &path); + else + strbuf_addstr(enlistment_root, the_repository->worktree); + } strbuf_release(&path); - setup_git_directory(); } static int run_git(const char *arg, ...) @@ -115,13 +86,39 @@ static int run_git(const char *arg, ...) return res; } +struct scalar_config { + const char *key; + const char *value; + int overwrite_on_reconfigure; +}; + +static int set_scalar_config(const struct scalar_config *config, int reconfigure) +{ + char *value = NULL; + int res; + + if ((reconfigure && config->overwrite_on_reconfigure) || + git_config_get_string(config->key, &value)) { + trace2_data_string("scalar", the_repository, config->key, "created"); + res = git_config_set_gently(config->key, config->value); + } else { + trace2_data_string("scalar", the_repository, config->key, "exists"); + res = 0; + } + + free(value); + return res; +} + +static int have_fsmonitor_support(void) +{ + return fsmonitor_ipc__is_supported() && + fsm_settings__get_reason(the_repository) == FSMONITOR_REASON_OK; +} + static int set_recommended_config(int reconfigure) { - struct { - const char *key; - const char *value; - int overwrite_on_reconfigure; - } config[] = { + struct scalar_config config[] = { /* Required */ { "am.keepCR", "true", 1 }, { "core.FSCache", "true", 1 }, @@ -175,17 +172,16 @@ static int set_recommended_config(int reconfigure) char *value; for (i = 0; config[i].key; i++) { - if ((reconfigure && config[i].overwrite_on_reconfigure) || - git_config_get_string(config[i].key, &value)) { - trace2_data_string("scalar", the_repository, config[i].key, "created"); - if (git_config_set_gently(config[i].key, - config[i].value) < 0) - return error(_("could not configure %s=%s"), - config[i].key, config[i].value); - } else { - trace2_data_string("scalar", the_repository, config[i].key, "exists"); - free(value); - } + if (set_scalar_config(config + i, reconfigure)) + return error(_("could not configure %s=%s"), + config[i].key, config[i].value); + } + + if (have_fsmonitor_support()) { + struct scalar_config fsmonitor = { "core.fsmonitor", "true" }; + if (set_scalar_config(&fsmonitor, reconfigure)) + return error(_("could not configure %s=%s"), + fsmonitor.key, fsmonitor.value); } /* @@ -236,123 +232,55 @@ static int add_or_remove_enlistment(int add) "scalar.repo", the_repository->worktree, NULL); } -static int register_dir(void) +static int start_fsmonitor_daemon(void) { - int res = add_or_remove_enlistment(1); - - if (!res) - res = set_recommended_config(0); + assert(have_fsmonitor_support()); - if (!res) - res = toggle_maintenance(1); + if (fsmonitor_ipc__get_state() != IPC_STATE__LISTENING) + return run_git("fsmonitor--daemon", "start", NULL); - return res; + return 0; } -static int unregister_dir(void) +static int stop_fsmonitor_daemon(void) { - int res = 0; + assert(have_fsmonitor_support()); - if (toggle_maintenance(0) < 0) - res = -1; + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + return run_git("fsmonitor--daemon", "stop", NULL); - if (add_or_remove_enlistment(0) < 0) - res = -1; - - return res; + return 0; } -static int add_directory_to_archiver(struct strvec *archiver_args, - const char *path, int recurse) +static int register_dir(void) { - int at_root = !*path; - DIR *dir = opendir(at_root ? "." : path); - struct dirent *e; - struct strbuf buf = STRBUF_INIT; - size_t len; - int res = 0; + if (add_or_remove_enlistment(1)) + return error(_("could not add enlistment")); - if (!dir) - return error_errno(_("could not open directory '%s'"), path); - - if (!at_root) - strbuf_addf(&buf, "%s/", path); - len = buf.len; - strvec_pushf(archiver_args, "--prefix=%s", buf.buf); - - while (!res && (e = readdir(dir))) { - if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name)) - continue; - - strbuf_setlen(&buf, len); - strbuf_addstr(&buf, e->d_name); - - if (e->d_type == DT_REG) - strvec_pushf(archiver_args, "--add-file=%s", buf.buf); - else if (e->d_type != DT_DIR) - warning(_("skipping '%s', which is neither file nor " - "directory"), buf.buf); - else if (recurse && - add_directory_to_archiver(archiver_args, - buf.buf, recurse) < 0) - res = -1; + if (set_recommended_config(0)) + return error(_("could not set recommended config")); + + if (toggle_maintenance(1)) + return error(_("could not turn on maintenance")); + + if (have_fsmonitor_support() && start_fsmonitor_daemon()) { + return error(_("could not start the FSMonitor daemon")); } - closedir(dir); - strbuf_release(&buf); - return res; + return 0; } -#ifndef WIN32 -#include <sys/statvfs.h> -#endif - -static int get_disk_info(struct strbuf *out) +static int unregister_dir(void) { -#ifdef WIN32 - struct strbuf buf = STRBUF_INIT; - char volume_name[MAX_PATH], fs_name[MAX_PATH]; - DWORD serial_number, component_length, flags; - ULARGE_INTEGER avail2caller, total, avail; - - strbuf_realpath(&buf, ".", 1); - if (!GetDiskFreeSpaceExA(buf.buf, &avail2caller, &total, &avail)) { - error(_("could not determine free disk size for '%s'"), - buf.buf); - strbuf_release(&buf); - return -1; - } + int res = 0; - strbuf_setlen(&buf, offset_1st_component(buf.buf)); - if (!GetVolumeInformationA(buf.buf, volume_name, sizeof(volume_name), - &serial_number, &component_length, &flags, - fs_name, sizeof(fs_name))) { - error(_("could not get info for '%s'"), buf.buf); - strbuf_release(&buf); - return -1; - } - strbuf_addf(out, "Available space on '%s': ", buf.buf); - strbuf_humanise_bytes(out, avail2caller.QuadPart); - strbuf_addch(out, '\n'); - strbuf_release(&buf); -#else - struct strbuf buf = STRBUF_INIT; - struct statvfs stat; + if (toggle_maintenance(0)) + res = error(_("could not turn off maintenance")); - strbuf_realpath(&buf, ".", 1); - if (statvfs(buf.buf, &stat) < 0) { - error_errno(_("could not determine free disk size for '%s'"), - buf.buf); - strbuf_release(&buf); - return -1; - } + if (add_or_remove_enlistment(0)) + res = error(_("could not remove enlistment")); - strbuf_addf(out, "Available space on '%s': ", buf.buf); - strbuf_humanise_bytes(out, st_mult(stat.f_bsize, stat.f_bavail)); - strbuf_addf(out, " (mount flags 0x%lx)\n", stat.f_flag); - strbuf_release(&buf); -#endif - return 0; + return res; } /* printf-style interface, expects `<key>=<value>` argument */ @@ -431,25 +359,35 @@ static int delete_enlistment(struct strbuf *enlistment) { #ifdef WIN32 struct strbuf parent = STRBUF_INIT; + size_t offset; + char *path_sep; #endif if (unregister_dir()) - die(_("failed to unregister repository")); + return error(_("failed to unregister repository")); #ifdef WIN32 /* * Change the current directory to one outside of the enlistment so * that we may delete everything underneath it. */ - strbuf_addbuf(&parent, enlistment); - strbuf_parent_directory(&parent); - if (chdir(parent.buf) < 0) - die_errno(_("could not switch to '%s'"), parent.buf); + offset = offset_1st_component(enlistment->buf); + path_sep = find_last_dir_sep(enlistment->buf + offset); + strbuf_add(&parent, enlistment->buf, + path_sep ? path_sep - enlistment->buf : offset); + if (chdir(parent.buf) < 0) { + int res = error_errno(_("could not switch to '%s'"), parent.buf); + strbuf_release(&parent); + return res; + } strbuf_release(&parent); #endif + if (have_fsmonitor_support() && stop_fsmonitor_daemon()) + return error(_("failed to stop the FSMonitor daemon")); + if (remove_dir_recursively(enlistment, 0)) - die(_("failed to delete enlistment directory")); + return error(_("failed to delete enlistment directory")); return 0; } @@ -595,83 +533,6 @@ cleanup: return res; } -static void dir_file_stats_objects(const char *full_path, size_t full_path_len, - const char *file_name, void *data) -{ - struct strbuf *buf = data; - struct stat st; - - if (!stat(full_path, &st)) - strbuf_addf(buf, "%-70s %16" PRIuMAX "\n", file_name, - (uintmax_t)st.st_size); -} - -static int dir_file_stats(struct object_directory *object_dir, void *data) -{ - struct strbuf *buf = data; - - strbuf_addf(buf, "Contents of %s:\n", object_dir->path); - - for_each_file_in_pack_dir(object_dir->path, dir_file_stats_objects, - data); - - return 0; -} - -static int count_files(char *path) -{ - DIR *dir = opendir(path); - struct dirent *e; - int count = 0; - - if (!dir) - return 0; - - while ((e = readdir(dir)) != NULL) - if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) - count++; - - closedir(dir); - return count; -} - -static void loose_objs_stats(struct strbuf *buf, const char *path) -{ - DIR *dir = opendir(path); - struct dirent *e; - int count; - int total = 0; - unsigned char c; - struct strbuf count_path = STRBUF_INIT; - size_t base_path_len; - - if (!dir) - return; - - strbuf_addstr(buf, "Object directory stats for "); - strbuf_add_absolute_path(buf, path); - strbuf_addstr(buf, ":\n"); - - strbuf_add_absolute_path(&count_path, path); - strbuf_addch(&count_path, '/'); - base_path_len = count_path.len; - - while ((e = readdir(dir)) != NULL) - if (!is_dot_or_dotdot(e->d_name) && - e->d_type == DT_DIR && strlen(e->d_name) == 2 && - !hex_to_bytes(&c, e->d_name, 1)) { - strbuf_setlen(&count_path, base_path_len); - strbuf_addstr(&count_path, e->d_name); - total += (count = count_files(count_path.buf)); - strbuf_addf(buf, "%s : %7d files\n", e->d_name, count); - } - - strbuf_addf(buf, "Total: %d loose objects", total); - - strbuf_release(&count_path); - closedir(dir); -} - static int cmd_diagnose(int argc, const char **argv) { struct option options[] = { @@ -681,106 +542,19 @@ static int cmd_diagnose(int argc, const char **argv) N_("scalar diagnose [<enlistment>]"), NULL }; - struct strbuf zip_path = STRBUF_INIT; - struct strvec archiver_args = STRVEC_INIT; - char **argv_copy = NULL; - int stdout_fd = -1, archiver_fd = -1; - time_t now = time(NULL); - struct tm tm; - struct strbuf buf = STRBUF_INIT; + struct strbuf diagnostics_root = STRBUF_INIT; int res = 0; argc = parse_options(argc, argv, NULL, options, usage, 0); - setup_enlistment_directory(argc, argv, usage, options, &zip_path); - - strbuf_addstr(&zip_path, "/.scalarDiagnostics/scalar_"); - strbuf_addftime(&zip_path, - "%Y%m%d_%H%M%S", localtime_r(&now, &tm), 0, 0); - strbuf_addstr(&zip_path, ".zip"); - switch (safe_create_leading_directories(zip_path.buf)) { - case SCLD_EXISTS: - case SCLD_OK: - break; - default: - error_errno(_("could not create directory for '%s'"), - zip_path.buf); - goto diagnose_cleanup; - } - stdout_fd = dup(1); - if (stdout_fd < 0) { - res = error_errno(_("could not duplicate stdout")); - goto diagnose_cleanup; - } - - archiver_fd = xopen(zip_path.buf, O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (archiver_fd < 0 || dup2(archiver_fd, 1) < 0) { - res = error_errno(_("could not redirect output")); - goto diagnose_cleanup; - } - - init_zip_archiver(); - strvec_pushl(&archiver_args, "scalar-diagnose", "--format=zip", NULL); + setup_enlistment_directory(argc, argv, usage, options, &diagnostics_root); + strbuf_addstr(&diagnostics_root, "/.scalarDiagnostics"); - strbuf_reset(&buf); - strbuf_addstr(&buf, "Collecting diagnostic info\n\n"); - get_version_info(&buf, 1); - - strbuf_addf(&buf, "Enlistment root: %s\n", the_repository->worktree); - get_disk_info(&buf); - write_or_die(stdout_fd, buf.buf, buf.len); - strvec_pushf(&archiver_args, - "--add-virtual-file=diagnostics.log:%.*s", - (int)buf.len, buf.buf); - - strbuf_reset(&buf); - strbuf_addstr(&buf, "--add-virtual-file=packs-local.txt:"); - dir_file_stats(the_repository->objects->odb, &buf); - foreach_alt_odb(dir_file_stats, &buf); - strvec_push(&archiver_args, buf.buf); - - strbuf_reset(&buf); - strbuf_addstr(&buf, "--add-virtual-file=objects-local.txt:"); - loose_objs_stats(&buf, ".git/objects"); - strvec_push(&archiver_args, buf.buf); - - if ((res = add_directory_to_archiver(&archiver_args, ".git", 0)) || - (res = add_directory_to_archiver(&archiver_args, ".git/hooks", 0)) || - (res = add_directory_to_archiver(&archiver_args, ".git/info", 0)) || - (res = add_directory_to_archiver(&archiver_args, ".git/logs", 1)) || - (res = add_directory_to_archiver(&archiver_args, ".git/objects/info", 0))) - goto diagnose_cleanup; - - strvec_pushl(&archiver_args, "--prefix=", - oid_to_hex(the_hash_algo->empty_tree), "--", NULL); - - /* `write_archive()` modifies the `argv` passed to it. Let it. */ - argv_copy = xmemdupz(archiver_args.v, - sizeof(char *) * archiver_args.nr); - res = write_archive(archiver_args.nr, (const char **)argv_copy, NULL, - the_repository, NULL, 0); - if (res) { - error(_("failed to write archive")); - goto diagnose_cleanup; - } - - if (!res) - fprintf(stderr, "\n" - "Diagnostics complete.\n" - "All of the gathered info is captured in '%s'\n", - zip_path.buf); - -diagnose_cleanup: - if (archiver_fd >= 0) { - close(1); - dup2(stdout_fd, 1); - } - free(argv_copy); - strvec_clear(&archiver_args); - strbuf_release(&zip_path); - strbuf_release(&buf); + res = run_git("diagnose", "--mode=all", "-s", "%Y%m%d_%H%M%S", + "-o", diagnostics_root.buf, NULL); + strbuf_release(&diagnostics_root); return res; } diff --git a/contrib/scalar/scalar.txt b/contrib/scalar/scalar.txt index c0425e0653..1a12dc4507 100644 --- a/contrib/scalar/scalar.txt +++ b/contrib/scalar/scalar.txt @@ -3,7 +3,7 @@ scalar(1) NAME ---- -scalar - an opinionated repository management tool +scalar - A tool for managing large Git repositories SYNOPSIS -------- @@ -20,10 +20,9 @@ scalar delete <enlistment> DESCRIPTION ----------- -Scalar is an opinionated repository management tool. By creating new -repositories or registering existing repositories with Scalar, your Git -experience will speed up. Scalar sets advanced Git config settings, -maintains your repositories in the background, and helps reduce data sent +Scalar is a repository management tool that optimizes Git for use in large +repositories. Scalar improves performance by configuring advanced Git settings, +maintaining repositories in the background, and helping to reduce data sent across the network. An important Scalar concept is the enlistment: this is the top-level directory diff --git a/contrib/scalar/t/Makefile b/contrib/scalar/t/Makefile index 01e82e56d1..1ed174a8cf 100644 --- a/contrib/scalar/t/Makefile +++ b/contrib/scalar/t/Makefile @@ -42,7 +42,7 @@ $(T): @echo "*** $@ ***"; GIT_CONFIG=.git/config '$(SHELL_PATH_SQ)' $@ $(GIT_TEST_OPTS) clean-except-prove-cache: - $(RM) -r 'trash directory'.* '$(TEST_RESULTS_DIRECTORY_SQ)' + $(RM) -r 'trash directory'.* $(RM) -r valgrind/bin clean: clean-except-prove-cache diff --git a/contrib/scalar/t/t9099-scalar.sh b/contrib/scalar/t/t9099-scalar.sh index 10b1172a8a..dfb949f52e 100755 --- a/contrib/scalar/t/t9099-scalar.sh +++ b/contrib/scalar/t/t9099-scalar.sh @@ -17,6 +17,99 @@ test_expect_success 'scalar shows a usage' ' test_expect_code 129 scalar -h ' +test_expect_success 'scalar invoked on enlistment root' ' + test_when_finished rm -rf test src deeper && + + for enlistment_root in test src deeper/test + do + git init ${enlistment_root}/src && + + # Register + scalar register ${enlistment_root} && + scalar list >out && + grep "$(pwd)/${enlistment_root}/src\$" out && + + # Delete (including enlistment root) + scalar delete $enlistment_root && + test_path_is_missing $enlistment_root && + scalar list >out && + ! grep "^$(pwd)/${enlistment_root}/src\$" out || return 1 + done +' + +test_expect_success 'scalar invoked on enlistment src repo' ' + test_when_finished rm -rf test src deeper && + + for enlistment_root in test src deeper/test + do + git init ${enlistment_root}/src && + + # Register + scalar register ${enlistment_root}/src && + scalar list >out && + grep "$(pwd)/${enlistment_root}/src\$" out && + + # Delete (will not include enlistment root) + scalar delete ${enlistment_root}/src && + test_path_is_dir $enlistment_root && + scalar list >out && + ! grep "^$(pwd)/${enlistment_root}/src\$" out || return 1 + done +' + +test_expect_success 'scalar invoked when enlistment root and repo are the same' ' + test_when_finished rm -rf test src deeper && + + for enlistment_root in test src deeper/test + do + git init ${enlistment_root} && + + # Register + scalar register ${enlistment_root} && + scalar list >out && + grep "$(pwd)/${enlistment_root}\$" out && + + # Delete (will not include enlistment root) + scalar delete ${enlistment_root} && + test_path_is_missing $enlistment_root && + scalar list >out && + ! grep "^$(pwd)/${enlistment_root}\$" out && + + # Make sure we did not accidentally delete the trash dir + test_path_is_dir "$TRASH_DIRECTORY" || return 1 + done +' + +test_expect_success 'scalar repo search respects GIT_CEILING_DIRECTORIES' ' + test_when_finished rm -rf test && + + git init test/src && + mkdir -p test/src/deep && + GIT_CEILING_DIRECTORIES="$(pwd)/test/src" && + ! scalar register test/src/deep 2>err && + grep "not a git repository" err +' + +test_expect_success 'scalar enlistments need a worktree' ' + test_when_finished rm -rf bare test && + + git init --bare bare/src && + ! scalar register bare/src 2>err && + grep "Scalar enlistments require a worktree" err && + + git init test/src && + ! scalar register test/src/.git 2>err && + grep "Scalar enlistments require a worktree" err +' + +test_expect_success FSMONITOR_DAEMON 'scalar register starts fsmon daemon' ' + git init test/src && + test_must_fail git -C test/src fsmonitor--daemon status && + scalar register test/src && + git -C test/src fsmonitor--daemon status && + test_cmp_config -C test/src true core.fsmonitor +' + test_expect_success 'scalar unregister' ' git init vanish/src && scalar register vanish/src && @@ -109,14 +202,14 @@ test_expect_success UNZIP 'scalar diagnose' ' sed -n "s/.*$SQ\\(.*\\.zip\\)$SQ.*/\\1/p" <err >zip_path && zip_path=$(cat zip_path) && test -n "$zip_path" && - unzip -v "$zip_path" && + "$GIT_UNZIP" -v "$zip_path" && folder=${zip_path%.zip} && test_path_is_missing "$folder" && - unzip -p "$zip_path" diagnostics.log >out && + "$GIT_UNZIP" -p "$zip_path" diagnostics.log >out && test_file_not_empty out && - unzip -p "$zip_path" packs-local.txt >out && + "$GIT_UNZIP" -p "$zip_path" packs-local.txt >out && grep "$(pwd)/.git/objects" out && - unzip -p "$zip_path" objects-local.txt >out && + "$GIT_UNZIP" -p "$zip_path" objects-local.txt >out && grep "^Total: [1-9]" out ' diff --git a/contrib/subtree/t/Makefile b/contrib/subtree/t/Makefile index 276898eb6b..3d278bb0ed 100644 --- a/contrib/subtree/t/Makefile +++ b/contrib/subtree/t/Makefile @@ -47,7 +47,7 @@ pre-clean: $(RM) -r '$(TEST_RESULTS_DIRECTORY_SQ)' clean-except-prove-cache: - $(RM) -r 'trash directory'.* '$(TEST_RESULTS_DIRECTORY_SQ)' + $(RM) -r 'trash directory'.* $(RM) -r valgrind/bin clean: clean-except-prove-cache @@ -53,7 +53,11 @@ struct delayed_checkout { enum ce_delay_state state; /* List of filter drivers that signaled delayed blobs. */ struct string_list filters; - /* List of delayed blobs identified by their path. */ + /* + * List of delayed blobs identified by their path. The `util` member + * holds a counter pointer which must be incremented when/if the + * associated blob gets checked out. + */ struct string_list paths; }; @@ -279,7 +279,7 @@ static const char *path_ok(const char *directory, struct hostinfo *hi) /* The validation is done on the paths after enter_repo * appends optional {.git,.git/.git} and friends, but * it does not use getcwd(). So if your /pub is - * a symlink to /mnt/pub, you can whitelist /pub and + * a symlink to /mnt/pub, you can include /pub and * do not have to say /mnt/pub. * Do not say /pub/. */ @@ -298,7 +298,7 @@ static const char *path_ok(const char *directory, struct hostinfo *hi) return path; } - logerror("'%s': not in whitelist", path); + logerror("'%s': not in directory list", path); return NULL; /* Fallthrough. Deny by default */ } @@ -403,7 +403,7 @@ static int run_service(const char *dir, struct daemon_service *service, * a "git-daemon-export-ok" flag that says that the other side * is ok with us doing this. * - * path_ok() uses enter_repo() and does whitelist checking. + * path_ok() uses enter_repo() and checks for included directories. * We only need to make sure the repository is exported. */ @@ -1444,7 +1444,7 @@ int cmd_main(int argc, const char **argv) cred = prepare_credentials(user_name, group_name); if (strict_paths && (!ok_paths || !*ok_paths)) - die("option --strict-paths requires a whitelist"); + die("option --strict-paths requires '<directory>' arguments"); if (base_path && !is_directory(base_path)) die("base-path '%s' does not exist or is not a directory", diff --git a/diagnose.c b/diagnose.c new file mode 100644 index 0000000000..beb0a8741b --- /dev/null +++ b/diagnose.c @@ -0,0 +1,269 @@ +#include "cache.h" +#include "diagnose.h" +#include "compat/disk.h" +#include "archive.h" +#include "dir.h" +#include "help.h" +#include "strvec.h" +#include "object-store.h" +#include "packfile.h" + +struct archive_dir { + const char *path; + int recursive; +}; + +struct diagnose_option { + enum diagnose_mode mode; + const char *option_name; +}; + +static struct diagnose_option diagnose_options[] = { + { DIAGNOSE_STATS, "stats" }, + { DIAGNOSE_ALL, "all" }, +}; + +int option_parse_diagnose(const struct option *opt, const char *arg, int unset) +{ + int i; + enum diagnose_mode *diagnose = opt->value; + + if (!arg) { + *diagnose = unset ? DIAGNOSE_NONE : DIAGNOSE_STATS; + return 0; + } + + for (i = 0; i < ARRAY_SIZE(diagnose_options); i++) { + if (!strcmp(arg, diagnose_options[i].option_name)) { + *diagnose = diagnose_options[i].mode; + return 0; + } + } + + return error(_("invalid --%s value '%s'"), opt->long_name, arg); +} + +static void dir_file_stats_objects(const char *full_path, size_t full_path_len, + const char *file_name, void *data) +{ + struct strbuf *buf = data; + struct stat st; + + if (!stat(full_path, &st)) + strbuf_addf(buf, "%-70s %16" PRIuMAX "\n", file_name, + (uintmax_t)st.st_size); +} + +static int dir_file_stats(struct object_directory *object_dir, void *data) +{ + struct strbuf *buf = data; + + strbuf_addf(buf, "Contents of %s:\n", object_dir->path); + + for_each_file_in_pack_dir(object_dir->path, dir_file_stats_objects, + data); + + return 0; +} + +static int count_files(char *path) +{ + DIR *dir = opendir(path); + struct dirent *e; + int count = 0; + + if (!dir) + return 0; + + while ((e = readdir(dir)) != NULL) + if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) + count++; + + closedir(dir); + return count; +} + +static void loose_objs_stats(struct strbuf *buf, const char *path) +{ + DIR *dir = opendir(path); + struct dirent *e; + int count; + int total = 0; + unsigned char c; + struct strbuf count_path = STRBUF_INIT; + size_t base_path_len; + + if (!dir) + return; + + strbuf_addstr(buf, "Object directory stats for "); + strbuf_add_absolute_path(buf, path); + strbuf_addstr(buf, ":\n"); + + strbuf_add_absolute_path(&count_path, path); + strbuf_addch(&count_path, '/'); + base_path_len = count_path.len; + + while ((e = readdir(dir)) != NULL) + if (!is_dot_or_dotdot(e->d_name) && + e->d_type == DT_DIR && strlen(e->d_name) == 2 && + !hex_to_bytes(&c, e->d_name, 1)) { + strbuf_setlen(&count_path, base_path_len); + strbuf_addstr(&count_path, e->d_name); + total += (count = count_files(count_path.buf)); + strbuf_addf(buf, "%s : %7d files\n", e->d_name, count); + } + + strbuf_addf(buf, "Total: %d loose objects", total); + + strbuf_release(&count_path); + closedir(dir); +} + +static int add_directory_to_archiver(struct strvec *archiver_args, + const char *path, int recurse) +{ + int at_root = !*path; + DIR *dir; + struct dirent *e; + struct strbuf buf = STRBUF_INIT; + size_t len; + int res = 0; + + dir = opendir(at_root ? "." : path); + if (!dir) { + if (errno == ENOENT) { + warning(_("could not archive missing directory '%s'"), path); + return 0; + } + return error_errno(_("could not open directory '%s'"), path); + } + + if (!at_root) + strbuf_addf(&buf, "%s/", path); + len = buf.len; + strvec_pushf(archiver_args, "--prefix=%s", buf.buf); + + while (!res && (e = readdir(dir))) { + if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name)) + continue; + + strbuf_setlen(&buf, len); + strbuf_addstr(&buf, e->d_name); + + if (e->d_type == DT_REG) + strvec_pushf(archiver_args, "--add-file=%s", buf.buf); + else if (e->d_type != DT_DIR) + warning(_("skipping '%s', which is neither file nor " + "directory"), buf.buf); + else if (recurse && + add_directory_to_archiver(archiver_args, + buf.buf, recurse) < 0) + res = -1; + } + + closedir(dir); + strbuf_release(&buf); + return res; +} + +int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode) +{ + struct strvec archiver_args = STRVEC_INIT; + char **argv_copy = NULL; + int stdout_fd = -1, archiver_fd = -1; + struct strbuf buf = STRBUF_INIT; + int res, i; + struct archive_dir archive_dirs[] = { + { ".git", 0 }, + { ".git/hooks", 0 }, + { ".git/info", 0 }, + { ".git/logs", 1 }, + { ".git/objects/info", 0 } + }; + + if (mode == DIAGNOSE_NONE) { + res = 0; + goto diagnose_cleanup; + } + + stdout_fd = dup(STDOUT_FILENO); + if (stdout_fd < 0) { + res = error_errno(_("could not duplicate stdout")); + goto diagnose_cleanup; + } + + archiver_fd = xopen(zip_path->buf, O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (dup2(archiver_fd, STDOUT_FILENO) < 0) { + res = error_errno(_("could not redirect output")); + goto diagnose_cleanup; + } + + init_zip_archiver(); + strvec_pushl(&archiver_args, "git-diagnose", "--format=zip", NULL); + + strbuf_reset(&buf); + strbuf_addstr(&buf, "Collecting diagnostic info\n\n"); + get_version_info(&buf, 1); + + strbuf_addf(&buf, "Repository root: %s\n", the_repository->worktree); + get_disk_info(&buf); + write_or_die(stdout_fd, buf.buf, buf.len); + strvec_pushf(&archiver_args, + "--add-virtual-file=diagnostics.log:%.*s", + (int)buf.len, buf.buf); + + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=packs-local.txt:"); + dir_file_stats(the_repository->objects->odb, &buf); + foreach_alt_odb(dir_file_stats, &buf); + strvec_push(&archiver_args, buf.buf); + + strbuf_reset(&buf); + strbuf_addstr(&buf, "--add-virtual-file=objects-local.txt:"); + loose_objs_stats(&buf, ".git/objects"); + strvec_push(&archiver_args, buf.buf); + + /* Only include this if explicitly requested */ + if (mode == DIAGNOSE_ALL) { + for (i = 0; i < ARRAY_SIZE(archive_dirs); i++) { + if (add_directory_to_archiver(&archiver_args, + archive_dirs[i].path, + archive_dirs[i].recursive)) { + res = error_errno(_("could not add directory '%s' to archiver"), + archive_dirs[i].path); + goto diagnose_cleanup; + } + } + } + + strvec_pushl(&archiver_args, "--prefix=", + oid_to_hex(the_hash_algo->empty_tree), "--", NULL); + + /* `write_archive()` modifies the `argv` passed to it. Let it. */ + argv_copy = xmemdupz(archiver_args.v, + sizeof(char *) * archiver_args.nr); + res = write_archive(archiver_args.nr, (const char **)argv_copy, NULL, + the_repository, NULL, 0); + if (res) { + error(_("failed to write archive")); + goto diagnose_cleanup; + } + + fprintf(stderr, "\n" + "Diagnostics complete.\n" + "All of the gathered info is captured in '%s'\n", + zip_path->buf); + +diagnose_cleanup: + if (archiver_fd >= 0) { + dup2(stdout_fd, STDOUT_FILENO); + close(stdout_fd); + close(archiver_fd); + } + free(argv_copy); + strvec_clear(&archiver_args); + strbuf_release(&buf); + + return res; +} diff --git a/diagnose.h b/diagnose.h new file mode 100644 index 0000000000..7a4951a786 --- /dev/null +++ b/diagnose.h @@ -0,0 +1,17 @@ +#ifndef DIAGNOSE_H +#define DIAGNOSE_H + +#include "strbuf.h" +#include "parse-options.h" + +enum diagnose_mode { + DIAGNOSE_NONE, + DIAGNOSE_STATS, + DIAGNOSE_ALL +}; + +int option_parse_diagnose(const struct option *opt, const char *arg, int unset); + +int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode); + +#endif /* DIAGNOSE_H */ diff --git a/diff-lib.c b/diff-lib.c index 7eb66a417a..2edea41a23 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -466,6 +466,11 @@ static void do_oneway_diff(struct unpack_trees_options *o, * Something removed from the tree? */ if (!idx) { + if (S_ISSPARSEDIR(tree->ce_mode)) { + diff_tree_oid(&tree->oid, NULL, tree->name, &revs->diffopt); + return; + } + diff_index_show_file(revs, "-", tree, &tree->oid, 1, tree->ce_mode, 0); return; @@ -1244,8 +1244,7 @@ int match_basename(const char *basename, int basenamelen, int match_pathname(const char *pathname, int pathlen, const char *base, int baselen, - const char *pattern, int prefix, int patternlen, - unsigned flags) + const char *pattern, int prefix, int patternlen) { const char *name; int namelen; @@ -1347,8 +1346,7 @@ static struct path_pattern *last_matching_pattern_from_list(const char *pathname if (match_pathname(pathname, pathlen, pattern->base, pattern->baselen ? pattern->baselen - 1 : 0, - exclude, prefix, pattern->patternlen, - pattern->flags)) { + exclude, prefix, pattern->patternlen)) { res = pattern; break; } @@ -414,7 +414,7 @@ int match_basename(const char *, int, const char *, int, int, unsigned); int match_pathname(const char *, int, const char *, int, - const char *, int, int, unsigned); + const char *, int, int); struct path_pattern *last_matching_pattern(struct dir_struct *dir, struct index_state *istate, @@ -157,12 +157,11 @@ static int remove_available_paths(struct string_list_item *item, void *cb_data) available = string_list_lookup(available_paths, item->string); if (available) - available->util = (void *)item->string; + available->util = item->util; return !available; } -int finish_delayed_checkout(struct checkout *state, int *nr_checkouts, - int show_progress) +int finish_delayed_checkout(struct checkout *state, int show_progress) { int errs = 0; unsigned processed_paths = 0; @@ -227,7 +226,7 @@ int finish_delayed_checkout(struct checkout *state, int *nr_checkouts, strlen(path->string), 0); if (ce) { display_progress(progress, ++processed_paths); - errs |= checkout_entry(ce, state, NULL, nr_checkouts); + errs |= checkout_entry(ce, state, NULL, path->util); filtered_bytes += ce->ce_stat_data.sd_size; display_throughput(progress, filtered_bytes); } else @@ -266,7 +265,8 @@ void update_ce_after_write(const struct checkout *state, struct cache_entry *ce, /* Note: ca is used (and required) iff the entry refers to a regular file. */ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca, - const struct checkout *state, int to_tempfile) + const struct checkout *state, int to_tempfile, + int *nr_checkouts) { unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT; struct delayed_checkout *dco = state->delayed_checkout; @@ -279,6 +279,7 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca struct stat st; const struct submodule *sub; struct checkout_metadata meta; + static int scratch_nr_checkouts; clone_checkout_metadata(&meta, &state->meta, &ce->oid); @@ -333,9 +334,15 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca ret = async_convert_to_working_tree_ca(ca, ce->name, new_blob, size, &buf, &meta, dco); - if (ret && string_list_has_string(&dco->paths, ce->name)) { - free(new_blob); - goto delayed; + if (ret) { + struct string_list_item *item = + string_list_lookup(&dco->paths, ce->name); + if (item) { + item->util = nr_checkouts ? nr_checkouts + : &scratch_nr_checkouts; + free(new_blob); + goto delayed; + } } } else { ret = convert_to_working_tree_ca(ca, ce->name, new_blob, @@ -392,6 +399,8 @@ finish: ce->name); update_ce_after_write(state, ce , &st); } + if (nr_checkouts) + (*nr_checkouts)++; delayed: return 0; } @@ -476,7 +485,7 @@ int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca, convert_attrs(state->istate, &ca_buf, ce->name); ca = &ca_buf; } - return write_entry(ce, topath, ca, state, 1); + return write_entry(ce, topath, ca, state, 1, nr_checkouts); } strbuf_reset(&path); @@ -540,18 +549,15 @@ int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca, create_directories(path.buf, path.len, state); - if (nr_checkouts) - (*nr_checkouts)++; - if (S_ISREG(ce->ce_mode) && !ca) { convert_attrs(state->istate, &ca_buf, ce->name); ca = &ca_buf; } - if (!enqueue_checkout(ce, ca)) + if (!enqueue_checkout(ce, ca, nr_checkouts)) return 0; - return write_entry(ce, path.buf, ca, state, 0); + return write_entry(ce, path.buf, ca, state, 0, nr_checkouts); } void unlink_entry(const struct cache_entry *ce) @@ -43,8 +43,7 @@ static inline int checkout_entry(struct cache_entry *ce, } void enable_delayed_checkout(struct checkout *state); -int finish_delayed_checkout(struct checkout *state, int *nr_checkouts, - int show_progress); +int finish_delayed_checkout(struct checkout *state, int show_progress); /* * Unlink the last component and schedule the leading directories for diff --git a/environment.c b/environment.c index b3296ce7d1..b2004437dc 100644 --- a/environment.c +++ b/environment.c @@ -56,7 +56,6 @@ const char *askpass_program; const char *excludes_file; enum auto_crlf auto_crlf = AUTO_CRLF_FALSE; int read_replace_refs = 1; -char *git_replace_ref_base; enum eol core_eol = EOL_UNSET; int global_conv_flags_eol = CONV_EOL_RNDTRP_WARN; char *check_roundtrip_encoding = "SHIFT-JIS"; @@ -162,6 +161,7 @@ const char *getenv_safe(struct strvec *argv, const char *name) void setup_git_env(const char *git_dir) { + char *git_replace_ref_base; const char *shallow_file; const char *replace_ref_base; struct set_gitdir_args args = { NULL }; @@ -182,9 +182,10 @@ void setup_git_env(const char *git_dir) if (getenv(NO_REPLACE_OBJECTS_ENVIRONMENT)) read_replace_refs = 0; replace_ref_base = getenv(GIT_REPLACE_REF_BASE_ENVIRONMENT); - free(git_replace_ref_base); git_replace_ref_base = xstrdup(replace_ref_base ? replace_ref_base : "refs/replace/"); + update_ref_namespace(NAMESPACE_REPLACE, git_replace_ref_base); + free(git_namespace); git_namespace = expand_namespace(getenv(GIT_NAMESPACE_ENVIRONMENT)); shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT); diff --git a/fetch-pack.c b/fetch-pack.c index cb6647d657..a1a508ee72 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -26,6 +26,7 @@ #include "commit-reach.h" #include "commit-graph.h" #include "sigchain.h" +#include "mergesort.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -292,6 +293,29 @@ static void mark_tips(struct fetch_negotiator *negotiator, return; } +static void send_filter(struct fetch_pack_args *args, + struct strbuf *req_buf, + int server_supports_filter) +{ + if (args->filter_options.choice) { + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); + if (server_supports_filter) { + print_verbose(args, _("Server supports filter")); + packet_buf_write(req_buf, "filter %s", spec); + trace2_data_string("fetch", the_repository, + "filter/effective", spec); + } else { + warning("filtering not recognized by server, ignoring"); + trace2_data_string("fetch", the_repository, + "filter/unsupported", spec); + } + } else { + trace2_data_string("fetch", the_repository, + "filter/none", ""); + } +} + static int find_common(struct fetch_negotiator *negotiator, struct fetch_pack_args *args, int fd[2], struct object_id *result_oid, @@ -299,6 +323,7 @@ static int find_common(struct fetch_negotiator *negotiator, { int fetching; int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval; + int negotiation_round = 0, haves = 0; const struct object_id *oid; unsigned in_vain = 0; int got_continue = 0; @@ -389,11 +414,7 @@ static int find_common(struct fetch_negotiator *negotiator, packet_buf_write(&req_buf, "deepen-not %s", s->string); } } - if (server_supports_filtering && args->filter_options.choice) { - const char *spec = - expand_list_objects_filter_spec(&args->filter_options); - packet_buf_write(&req_buf, "filter %s", spec); - } + send_filter(args, &req_buf, server_supports_filtering); packet_buf_flush(&req_buf); state_len = req_buf.len; @@ -441,9 +462,19 @@ static int find_common(struct fetch_negotiator *negotiator, packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid)); print_verbose(args, "have %s", oid_to_hex(oid)); in_vain++; + haves++; if (flush_at <= ++count) { int ack; + negotiation_round++; + trace2_region_enter_printf("negotiation_v0_v1", "round", + the_repository, "%d", + negotiation_round); + trace2_data_intmax("negotiation_v0_v1", the_repository, + "haves_added", haves); + trace2_data_intmax("negotiation_v0_v1", the_repository, + "in_vain", in_vain); + haves = 0; packet_buf_flush(&req_buf); send_request(args, fd[1], &req_buf); strbuf_setlen(&req_buf, state_len); @@ -465,6 +496,9 @@ static int find_common(struct fetch_negotiator *negotiator, ack, oid_to_hex(result_oid)); switch (ack) { case ACK: + trace2_region_leave_printf("negotiation_v0_v1", "round", + the_repository, "%d", + negotiation_round); flushes = 0; multi_ack = 0; retval = 0; @@ -490,6 +524,7 @@ static int find_common(struct fetch_negotiator *negotiator, const char *hex = oid_to_hex(result_oid); packet_buf_write(&req_buf, "have %s\n", hex); state_len = req_buf.len; + haves++; /* * Reset in_vain because an ack * for this commit has not been @@ -508,6 +543,9 @@ static int find_common(struct fetch_negotiator *negotiator, } } while (ack); flushes--; + trace2_region_leave_printf("negotiation_v0_v1", "round", + the_repository, "%d", + negotiation_round); if (got_continue && MAX_IN_VAIN < in_vain) { print_verbose(args, _("giving up")); break; /* give up */ @@ -518,6 +556,8 @@ static int find_common(struct fetch_negotiator *negotiator, } done: trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository); + trace2_data_intmax("negotiation_v0_v1", the_repository, "total_rounds", + negotiation_round); if (!got_ready || !no_done) { packet_buf_write(&req_buf, "done\n"); send_request(args, fd[1], &req_buf); @@ -1025,6 +1065,13 @@ static int get_pack(struct fetch_pack_args *args, return 0; } +static int ref_compare_name(const struct ref *a, const struct ref *b) +{ + return strcmp(a->name, b->name); +} + +DEFINE_LIST_SORT(static, sort_ref_list, struct ref, next); + static int cmp_ref_by_name(const void *a_, const void *b_) { const struct ref *a = *((const struct ref **)a_); @@ -1323,15 +1370,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, die(_("Server does not support shallow requests")); /* Add filter */ - if (server_supports_feature("fetch", "filter", 0) && - args->filter_options.choice) { - const char *spec = - expand_list_objects_filter_spec(&args->filter_options); - print_verbose(args, _("Server supports filter")); - packet_buf_write(&req_buf, "filter %s", spec); - } else if (args->filter_options.choice) { - warning("filtering not recognized by server, ignoring"); - } + send_filter(args, &req_buf, + server_supports_feature("fetch", "filter", 0)); if (server_supports_feature("fetch", "packfile-uris", 0)) { int i; @@ -1361,6 +1401,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, haves_added = add_haves(negotiator, &req_buf, haves_to_send); *in_vain += haves_added; + trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added); + trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain); if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) { /* Send Done */ packet_buf_write(&req_buf, "done\n"); @@ -1603,6 +1645,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, struct oidset common = OIDSET_INIT; struct packet_reader reader; int in_vain = 0, negotiation_started = 0; + int negotiation_round = 0; int haves_to_send = INITIAL_FLUSH; struct fetch_negotiator negotiator_alloc; struct fetch_negotiator *negotiator; @@ -1659,12 +1702,20 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, "negotiation_v2", the_repository); } + negotiation_round++; + trace2_region_enter_printf("negotiation_v2", "round", + the_repository, "%d", + negotiation_round); if (send_fetch_request(negotiator, fd[1], args, ref, &common, &haves_to_send, &in_vain, reader.use_sideband, - seen_ack)) + seen_ack)) { + trace2_region_leave_printf("negotiation_v2", "round", + the_repository, "%d", + negotiation_round); state = FETCH_GET_PACK; + } else state = FETCH_PROCESS_ACKS; break; @@ -1677,6 +1728,9 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, seen_ack = 1; oidset_insert(&common, &common_oid); } + trace2_region_leave_printf("negotiation_v2", "round", + the_repository, "%d", + negotiation_round); if (received_ready) { /* * Don't check for response delimiter; get_pack() will @@ -1692,6 +1746,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, trace2_region_leave("fetch-pack", "negotiation_v2", the_repository); + trace2_data_intmax("negotiation_v2", the_repository, + "total_rounds", negotiation_round); /* Check for shallow-info section */ if (process_section_header(&reader, "shallow-info", 1)) receive_shallow_info(args, &reader, shallows, si); @@ -2071,6 +2127,7 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, int in_vain = 0; int seen_ack = 0; int last_iteration = 0; + int negotiation_round = 0; timestamp_t min_generation = GENERATION_NUMBER_INFINITY; fetch_negotiator_init(the_repository, &negotiator); @@ -2084,11 +2141,17 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, add_to_object_array, &nt_object_array); + trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository); while (!last_iteration) { int haves_added; struct object_id common_oid; int received_ready = 0; + negotiation_round++; + + trace2_region_enter_printf("negotiate_using_fetch", "round", + the_repository, "%d", + negotiation_round); strbuf_reset(&req_buf); write_fetch_command_and_capabilities(&req_buf, server_options); @@ -2099,6 +2162,11 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN)) last_iteration = 1; + trace2_data_intmax("negotiate_using_fetch", the_repository, + "haves_added", haves_added); + trace2_data_intmax("negotiate_using_fetch", the_repository, + "in_vain", in_vain); + /* Send request */ packet_buf_flush(&req_buf); if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0) @@ -2131,7 +2199,13 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, REACH_SCRATCH, 0, min_generation)) last_iteration = 1; + trace2_region_leave_printf("negotiation", "round", + the_repository, "%d", + negotiation_round); } + trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository); + trace2_data_intmax("negotiate_using_fetch", the_repository, + "total_rounds", negotiation_round); clear_common_flag(acked_commits); strbuf_release(&req_buf); } @@ -308,7 +308,7 @@ static int fsck_walk_tree(struct tree *tree, void *data, struct fsck_options *op return -1; name = fsck_get_object_name(options, &tree->object.oid); - if (init_tree_desc_gently(&desc, tree->buffer, tree->size)) + if (init_tree_desc_gently(&desc, tree->buffer, tree->size, 0)) return -1; while (tree_entry_gently(&desc, &entry)) { struct object *obj; @@ -578,7 +578,7 @@ static int fsck_tree(const struct object_id *tree_oid, const char *o_name; struct name_stack df_dup_candidates = { NULL }; - if (init_tree_desc_gently(&desc, buffer, size)) { + if (init_tree_desc_gently(&desc, buffer, size, TREE_DESC_RAW_MODES)) { retval += report(options, tree_oid, OBJ_TREE, FSCK_MSG_BAD_TREE, "cannot be parsed as a tree"); @@ -56,7 +56,6 @@ enum fsck_msg_type { FUNC(GITMODULES_PATH, ERROR) \ FUNC(GITMODULES_UPDATE, ERROR) \ /* warnings */ \ - FUNC(BAD_FILEMODE, WARN) \ FUNC(EMPTY_NAME, WARN) \ FUNC(FULL_PATHNAME, WARN) \ FUNC(HAS_DOT, WARN) \ @@ -66,6 +65,7 @@ enum fsck_msg_type { FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \ /* infos (reported as warnings, but ignored by default) */ \ + FUNC(BAD_FILEMODE, INFO) \ FUNC(GITMODULES_PARSE, INFO) \ FUNC(GITIGNORE_SYMLINK, INFO) \ FUNC(GITATTRIBUTES_SYMLINK, INFO) \ diff --git a/fuzz-commit-graph.c b/fuzz-commit-graph.c index e7cf6d5b0f..914026f5d8 100644 --- a/fuzz-commit-graph.c +++ b/fuzz-commit-graph.c @@ -1,7 +1,7 @@ #include "commit-graph.h" #include "repository.h" -struct commit_graph *parse_commit_graph(struct repository *r, +struct commit_graph *parse_commit_graph(struct repo_settings *s, void *graph_map, size_t graph_size); int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); @@ -11,7 +11,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) struct commit_graph *g; initialize_the_repository(); - g = parse_commit_graph(the_repository, (void *)data, size); + /* + * Initialize the_repository with commit-graph settings that would + * normally be read from the repository's gitdir. We want to avoid + * touching the disk to keep the individual fuzz-test cases as fast as + * possible. + */ + the_repository->settings.commit_graph_generation_version = 2; + the_repository->settings.commit_graph_read_changed_paths = 1; + g = parse_commit_graph(&the_repository->settings, (void *)data, size); repo_clear(the_repository); free_commit_graph(g); diff --git a/git-compat-util.h b/git-compat-util.h index 58d7708296..4e51a1c48b 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -23,6 +23,9 @@ #include <crtdbg.h> #endif +struct strbuf; + + #define _FILE_OFFSET_BITS 64 @@ -258,6 +261,7 @@ static inline int is_xplatform_dir_sep(int c) #include <sys/resource.h> #include <sys/socket.h> #include <sys/ioctl.h> +#include <sys/statvfs.h> #include <termios.h> #ifndef NO_SYS_SELECT_H #include <sys/select.h> @@ -487,7 +491,7 @@ static inline void extract_id_from_env(const char *env, uid_t *id) } } -static inline int is_path_owned_by_current_uid(const char *path) +static inline int is_path_owned_by_current_uid(const char *path, struct strbuf *report) { struct stat st; uid_t euid; @@ -995,6 +999,28 @@ static inline unsigned long cast_size_t_to_ulong(size_t a) return (unsigned long)a; } +/* + * Limit size of IO chunks, because huge chunks only cause pain. OS X + * 64-bit is buggy, returning EINVAL if len >= INT_MAX; and even in + * the absence of bugs, large chunks can result in bad latencies when + * you decide to kill the process. + * + * We pick 8 MiB as our default, but if the platform defines SSIZE_MAX + * that is smaller than that, clip it to SSIZE_MAX, as a call to + * read(2) or write(2) larger than that is allowed to fail. As the last + * resort, we allow a port to pass via CFLAGS e.g. "-DMAX_IO_SIZE=value" + * to override this, if the definition of SSIZE_MAX given by the platform + * is broken. + */ +#ifndef MAX_IO_SIZE +# define MAX_IO_SIZE_DEFAULT (8*1024*1024) +# if defined(SSIZE_MAX) && (SSIZE_MAX < MAX_IO_SIZE_DEFAULT) +# define MAX_IO_SIZE SSIZE_MAX +# else +# define MAX_IO_SIZE MAX_IO_SIZE_DEFAULT +# endif +#endif + #ifdef HAVE_ALLOCA_H # include <alloca.h> # define xalloca(size) (alloca(size)) diff --git a/git-cvsserver.perl b/git-cvsserver.perl index 4c8118010a..7b757360e2 100755 --- a/git-cvsserver.perl +++ b/git-cvsserver.perl @@ -152,7 +152,7 @@ $state->{allowed_roots} = [ @ARGV ]; # don't export the whole system unless the users requests it if ($state->{'export-all'} && !@{$state->{allowed_roots}}) { - die "--export-all can only be used together with an explicit whitelist\n"; + die "--export-all can only be used together with an explicit '<directory>...' list\n"; } # Environment handling for running under git-shell diff --git a/git-merge-resolve.sh b/git-merge-resolve.sh index 343fe7bccd..77e93121bf 100755 --- a/git-merge-resolve.sh +++ b/git-merge-resolve.sh @@ -5,6 +5,16 @@ # # Resolve two trees, using enhanced multi-base read-tree. +. git-sh-setup + +# Abort if index does not match HEAD +if ! git diff-index --quiet --cached HEAD -- +then + gettextln "Error: Your local changes to the following files would be overwritten by merge" + git diff-index --cached --name-only HEAD -- | sed -e 's/^/ /' + exit 2 +fi + # The first parameters up to -- are merge bases; the rest are heads. bases= head= remotes= sep_seen= for arg @@ -822,6 +822,42 @@ def isModeExecChanged(src_mode, dst_mode): return isModeExec(src_mode) != isModeExec(dst_mode) +def p4KeysContainingNonUtf8Chars(): + """Returns all keys which may contain non UTF-8 encoded strings + for which a fallback strategy has to be applied. + """ + return ['desc', 'client', 'FullName'] + + +def p4KeysContainingBinaryData(): + """Returns all keys which may contain arbitrary binary data + """ + return ['data'] + + +def p4KeyContainsFilePaths(key): + """Returns True if the key contains file paths. These are handled by decode_path(). + Otherwise False. + """ + return key.startswith('depotFile') or key in ['path', 'clientFile'] + + +def p4KeyWhichCanBeDirectlyDecoded(key): + """Returns True if the key can be directly decoded as UTF-8 string + Otherwise False. + + Keys which can not be encoded directly: + - `data` which may contain arbitrary binary data + - `desc` or `client` or `FullName` which may contain non-UTF8 encoded text + - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path() + """ + if key in p4KeysContainingNonUtf8Chars() or \ + key in p4KeysContainingBinaryData() or \ + p4KeyContainsFilePaths(key): + return False + return True + + def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, errors_as_exceptions=False, *k, **kw): @@ -851,15 +887,13 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, try: while True: entry = marshal.load(p4.stdout) + if bytes is not str: - # Decode unmarshalled dict to use str keys and values, except for: - # - `data` which may contain arbitrary binary data - # - `desc` or `FullName` which may contain non-UTF8 encoded text handled below, eagerly converted to bytes - # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text, handled by decode_path() + # Decode unmarshalled dict to use str keys and values. Special cases are handled below. decoded_entry = {} for key, value in entry.items(): key = key.decode() - if isinstance(value, bytes) and not (key in ('data', 'desc', 'FullName', 'path', 'clientFile') or key.startswith('depotFile')): + if isinstance(value, bytes) and p4KeyWhichCanBeDirectlyDecoded(key): value = value.decode() decoded_entry[key] = value # Parse out data if it's an error response @@ -869,10 +903,9 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, if skip_info: if 'code' in entry and entry['code'] == 'info': continue - if 'desc' in entry: - entry['desc'] = metadata_stream_to_writable_bytes(entry['desc']) - if 'FullName' in entry: - entry['FullName'] = metadata_stream_to_writable_bytes(entry['FullName']) + for key in p4KeysContainingNonUtf8Chars(): + if key in entry: + entry[key] = metadata_stream_to_writable_bytes(entry[key]) if cb is not None: cb(entry) else: @@ -2226,7 +2259,7 @@ class P4Submit(Command, P4UserMap): raw=True): if regexp.search(line): if verbose: - print("got keyword match on %s in %s in %s" % (regex.pattern, line, file)) + print("got keyword match on %s in %s in %s" % (regexp.pattern, line, file)) kwfiles[file] = regexp break @@ -3148,7 +3181,7 @@ class P4Sync(Command, P4UserMap): raise e else: if p4_version_string().find('/NT') >= 0: - text = text.replace(b'\r\n', b'\n') + text = text.replace(b'\x0d\x00\x0a\x00', b'\x0a\x00') contents = [text] if type_base == "apple": @@ -4369,19 +4402,16 @@ class P4Unshelve(Command): def renameBranch(self, branch_name): """Rename the existing branch to branch_name.N .""" - found = True for i in range(0, 1000): backup_branch_name = "{0}.{1}".format(branch_name, i) if not gitBranchExists(backup_branch_name): # Copy ref to backup gitUpdateRef(backup_branch_name, branch_name) gitDeleteRef(branch_name) - found = True print("renamed old unshelve branch to {0}".format(backup_branch_name)) break - - if not found: - sys.exit("gave up trying to rename existing branch {0}".format(sync.branch)) + else: + sys.exit("gave up trying to rename existing branch {0}".format(branch_name)) def findLastP4Revision(self, starting_point): """Look back from starting_point for the first commit created by git-p4 @@ -522,6 +522,7 @@ static struct cmd_struct commands[] = { { "credential-cache--daemon", cmd_credential_cache_daemon }, { "credential-store", cmd_credential_store }, { "describe", cmd_describe, RUN_SETUP }, + { "diagnose", cmd_diagnose, RUN_SETUP_GENTLY }, { "diff", cmd_diff, NO_PARSEOPT }, { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE | NO_PARSEOPT }, { "diff-index", cmd_diff_index, RUN_SETUP | NO_PARSEOPT }, diff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl index 1835487ab2..e66eb3d9ba 100755 --- a/gitweb/gitweb.perl +++ b/gitweb/gitweb.perl @@ -3560,23 +3560,6 @@ sub parse_commit_text { $title =~ s/^ //; if ($title ne "") { $co{'title'} = chop_str($title, 80, 5); - # remove leading stuff of merges to make the interesting part visible - if (length($title) > 50) { - $title =~ s/^Automatic //; - $title =~ s/^merge (of|with) /Merge ... /i; - if (length($title) > 50) { - $title =~ s/(http|rsync):\/\///; - } - if (length($title) > 50) { - $title =~ s/(master|www|rsync)\.//; - } - if (length($title) > 50) { - $title =~ s/kernel.org:?//; - } - if (length($title) > 50) { - $title =~ s/\/pub\/scm//; - } - } $co{'title_short'} = chop_str($title, 50, 5); last; } @@ -38,19 +38,30 @@ static struct category_description main_categories[] = { { CAT_plumbinginterrogators, N_("Low-level Commands / Interrogators") }, { CAT_synchingrepositories, N_("Low-level Commands / Syncing Repositories") }, { CAT_purehelpers, N_("Low-level Commands / Internal Helpers") }, + { CAT_userinterfaces, N_("User-facing repository, command and file interfaces") }, + { CAT_developerinterfaces, N_("Developer-facing file file formats, protocols and interfaces") }, { 0, NULL } }; static const char *drop_prefix(const char *name, uint32_t category) { const char *new_name; - - if (skip_prefix(name, "git-", &new_name)) - return new_name; - if (category == CAT_guide && skip_prefix(name, "git", &new_name)) + const char *prefix; + + switch (category) { + case CAT_guide: + case CAT_userinterfaces: + case CAT_developerinterfaces: + prefix = "git"; + break; + default: + prefix = "git-"; + break; + } + if (skip_prefix(name, prefix, &new_name)) return new_name; - return name; + return name; } static void extract_cmds(struct cmdname_help **p_cmds, uint32_t mask) @@ -426,6 +437,26 @@ void list_guides_help(void) putchar('\n'); } +void list_user_interfaces_help(void) +{ + struct category_description catdesc[] = { + { CAT_userinterfaces, N_("User-facing repository, command and file interfaces:") }, + { 0, NULL } + }; + print_cmd_by_category(catdesc, NULL); + putchar('\n'); +} + +void list_developer_interfaces_help(void) +{ + struct category_description catdesc[] = { + { CAT_developerinterfaces, N_("File formats, protocols and other developer interfaces:") }, + { 0, NULL } + }; + print_cmd_by_category(catdesc, NULL); + putchar('\n'); +} + static int get_alias(const char *var, const char *value, void *data) { struct string_list *list = data; @@ -22,6 +22,8 @@ static inline void mput_char(char c, unsigned int num) void list_common_cmds_help(void); void list_all_cmds_help(int show_external_commands, int show_aliases); void list_guides_help(void); +void list_user_interfaces_help(void); +void list_developer_interfaces_help(void); void list_all_main_cmds(struct string_list *list); void list_all_other_cmds(struct string_list *list); @@ -62,9 +62,6 @@ static int pick_next_hook(struct child_process *cp, strvec_push(&cp->args, hook_path); strvec_pushv(&cp->args, hook_cb->options->args.v); - /* Provide context for errors if necessary */ - *pp_task_cb = (char *)hook_path; - /* * This pick_next_hook() will be called again, we're only * running one hook, so indicate that no more work will be @@ -80,13 +77,9 @@ static int notify_start_failure(struct strbuf *out, void *pp_task_cp) { struct hook_cb_data *hook_cb = pp_cb; - const char *hook_path = pp_task_cp; hook_cb->rc |= 1; - strbuf_addf(out, _("Couldn't start hook '%s'\n"), - hook_path); - return 1; } @@ -8,6 +8,7 @@ #include "cache.h" #include "config.h" #include "date.h" +#include "mailmap.h" static struct strbuf git_default_name = STRBUF_INIT; static struct strbuf git_default_email = STRBUF_INIT; @@ -346,6 +347,79 @@ person_only: return 0; } +/* + * Returns the difference between the new and old length of the ident line. + */ +static ssize_t rewrite_ident_line(const char *person, size_t len, + struct strbuf *buf, + struct string_list *mailmap) +{ + size_t namelen, maillen; + const char *name; + const char *mail; + struct ident_split ident; + + if (split_ident_line(&ident, person, len)) + return 0; + + mail = ident.mail_begin; + maillen = ident.mail_end - ident.mail_begin; + name = ident.name_begin; + namelen = ident.name_end - ident.name_begin; + + if (map_user(mailmap, &mail, &maillen, &name, &namelen)) { + struct strbuf namemail = STRBUF_INIT; + size_t newlen; + + strbuf_addf(&namemail, "%.*s <%.*s>", + (int)namelen, name, (int)maillen, mail); + + strbuf_splice(buf, ident.name_begin - buf->buf, + ident.mail_end - ident.name_begin + 1, + namemail.buf, namemail.len); + newlen = namemail.len; + + strbuf_release(&namemail); + + return newlen - (ident.mail_end - ident.name_begin); + } + + return 0; +} + +void apply_mailmap_to_header(struct strbuf *buf, const char **header, + struct string_list *mailmap) +{ + size_t buf_offset = 0; + + if (!mailmap) + return; + + for (;;) { + const char *person, *line; + size_t i; + int found_header = 0; + + line = buf->buf + buf_offset; + if (!*line || *line == '\n') + return; /* End of headers */ + + for (i = 0; header[i]; i++) + if (skip_prefix(line, header[i], &person)) { + const char *endp = strchrnul(person, '\n'); + found_header = 1; + buf_offset += endp - line; + buf_offset += rewrite_ident_line(person, endp - person, buf, mailmap); + break; + } + + if (!found_header) { + buf_offset = strchrnul(line, '\n') - buf->buf; + if (buf->buf[buf_offset] == '\n') + buf_offset++; + } + } +} static void ident_env_hint(enum want_ident whose_ident) { diff --git a/log-tree.c b/log-tree.c index d0ac0a6327..3e8c70ddcf 100644 --- a/log-tree.c +++ b/log-tree.c @@ -137,10 +137,12 @@ static int ref_filter_match(const char *refname, static int add_ref_decoration(const char *refname, const struct object_id *oid, int flags, void *cb_data) { + int i; struct object *obj; enum object_type objtype; enum decoration_type deco_type = DECORATION_NONE; struct decoration_filter *filter = (struct decoration_filter *)cb_data; + const char *git_replace_ref_base = ref_namespace[NAMESPACE_REPLACE].ref; if (filter && !ref_filter_match(refname, filter)) return 0; @@ -165,16 +167,21 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid, return 0; obj = lookup_object_by_type(the_repository, oid, objtype); - if (starts_with(refname, "refs/heads/")) - deco_type = DECORATION_REF_LOCAL; - else if (starts_with(refname, "refs/remotes/")) - deco_type = DECORATION_REF_REMOTE; - else if (starts_with(refname, "refs/tags/")) - deco_type = DECORATION_REF_TAG; - else if (!strcmp(refname, "refs/stash")) - deco_type = DECORATION_REF_STASH; - else if (!strcmp(refname, "HEAD")) - deco_type = DECORATION_REF_HEAD; + for (i = 0; i < ARRAY_SIZE(ref_namespace); i++) { + struct ref_namespace_info *info = &ref_namespace[i]; + + if (!info->decoration) + continue; + if (info->exact) { + if (!strcmp(refname, info->ref)) { + deco_type = info->decoration; + break; + } + } else if (starts_with(refname, info->ref)) { + deco_type = info->decoration; + break; + } + } add_name_decoration(deco_type, refname, obj); while (obj->type == OBJ_TAG) { @@ -956,8 +963,7 @@ static void cleanup_additional_headers(struct diff_options *o) static int do_remerge_diff(struct rev_info *opt, struct commit_list *parents, - struct object_id *oid, - struct commit *commit) + struct object_id *oid) { struct merge_options o; struct commit_list *bases; @@ -1052,7 +1058,7 @@ static int log_tree_diff(struct rev_info *opt, struct commit *commit, struct log "for octopus merges.\n"); return 1; } - return do_remerge_diff(opt, parents, oid, commit); + return do_remerge_diff(opt, parents, oid); } if (opt->combine_merges) return do_diff_combined(opt, commit); diff --git a/merge-ort-wrappers.c b/merge-ort-wrappers.c index ad04106169..748924a69b 100644 --- a/merge-ort-wrappers.c +++ b/merge-ort-wrappers.c @@ -10,8 +10,8 @@ static int unclean(struct merge_options *opt, struct tree *head) struct strbuf sb = STRBUF_INIT; if (head && repo_index_has_changes(opt->repo, head, &sb)) { - fprintf(stderr, _("Your local changes to the following files would be overwritten by merge:\n %s"), - sb.buf); + error(_("Your local changes to the following files would be overwritten by merge:\n %s"), + sb.buf); strbuf_release(&sb); return -1; } diff --git a/merge-ort.c b/merge-ort.c index 8b7de0fbd8..99dcee2db8 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -387,8 +387,24 @@ struct merge_options_internal { /* call_depth: recursion level counter for merging merge bases */ int call_depth; + + /* field that holds submodule conflict information */ + struct string_list conflicted_submodules; +}; + +struct conflicted_submodule_item { + char *abbrev; + int flag; }; +static void conflicted_submodule_item_free(void *util, const char *str) +{ + struct conflicted_submodule_item *item = util; + + free(item->abbrev); + free(item); +} + struct version_info { struct object_id oid; unsigned short mode; @@ -491,7 +507,6 @@ enum conflict_and_info_types { CONFLICT_FILE_DIRECTORY, CONFLICT_DISTINCT_MODES, CONFLICT_MODIFY_DELETE, - CONFLICT_PRESENT_DESPITE_SKIPPED, /* Regular rename */ CONFLICT_RENAME_RENAME, /* same file renamed differently */ @@ -517,6 +532,7 @@ enum conflict_and_info_types { CONFLICT_SUBMODULE_NOT_INITIALIZED, CONFLICT_SUBMODULE_HISTORY_NOT_AVAILABLE, CONFLICT_SUBMODULE_MAY_HAVE_REWINDS, + CONFLICT_SUBMODULE_NULL_MERGE_BASE, /* Keep this entry _last_ in the list */ NB_CONFLICT_TYPES, @@ -536,8 +552,6 @@ static const char *type_short_descriptions[] = { [CONFLICT_FILE_DIRECTORY] = "CONFLICT (file/directory)", [CONFLICT_DISTINCT_MODES] = "CONFLICT (distinct modes)", [CONFLICT_MODIFY_DELETE] = "CONFLICT (modify/delete)", - [CONFLICT_PRESENT_DESPITE_SKIPPED] = - "CONFLICT (upgrade your version of git)", /*** Regular rename ***/ [CONFLICT_RENAME_RENAME] = "CONFLICT (rename/rename)", @@ -570,6 +584,8 @@ static const char *type_short_descriptions[] = { "CONFLICT (submodule history not available)", [CONFLICT_SUBMODULE_MAY_HAVE_REWINDS] = "CONFLICT (submodule may have rewinds)", + [CONFLICT_SUBMODULE_NULL_MERGE_BASE] = + "CONFLICT (submodule lacks merge base)" }; struct logical_conflict_info { @@ -686,6 +702,9 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, mem_pool_discard(&opti->pool, 0); + string_list_clear_func(&opti->conflicted_submodules, + conflicted_submodule_item_free); + /* Clean out callback_data as well. */ FREE_AND_NULL(renames->callback_data); renames->callback_data_nr = renames->callback_data_alloc = 0; @@ -748,8 +767,7 @@ static void path_msg(struct merge_options *opt, /* Sanity checks */ assert(omittable_hint == !starts_with(type_short_descriptions[type], "CONFLICT") || - type == CONFLICT_DIR_RENAME_SUGGESTED || - type == CONFLICT_PRESENT_DESPITE_SKIPPED); + type == CONFLICT_DIR_RENAME_SUGGESTED); if (opt->record_conflict_msgs_as_headers && omittable_hint) return; /* Do not record mere hints in headers */ if (opt->priv->call_depth && opt->verbosity < 5) @@ -1744,24 +1762,32 @@ static int merge_submodule(struct merge_options *opt, int i; int search = !opt->priv->call_depth; + int sub_not_initialized = 1; + int sub_flag = CONFLICT_SUBMODULE_FAILED_TO_MERGE; /* store fallback answer in result in case we fail */ oidcpy(result, opt->priv->call_depth ? o : a); /* we can not handle deletion conflicts */ - if (is_null_oid(o)) - return 0; - if (is_null_oid(a)) - return 0; - if (is_null_oid(b)) - return 0; + if (is_null_oid(a) || is_null_oid(b)) + BUG("submodule deleted on one side; this should be handled outside of merge_submodule()"); - if (repo_submodule_init(&subrepo, opt->repo, path, null_oid())) { + if ((sub_not_initialized = repo_submodule_init(&subrepo, + opt->repo, path, null_oid()))) { path_msg(opt, CONFLICT_SUBMODULE_NOT_INITIALIZED, 0, path, NULL, NULL, NULL, _("Failed to merge submodule %s (not checked out)"), path); - return 0; + sub_flag = CONFLICT_SUBMODULE_NOT_INITIALIZED; + goto cleanup; + } + + if (is_null_oid(o)) { + path_msg(opt, CONFLICT_SUBMODULE_NULL_MERGE_BASE, 0, + path, NULL, NULL, NULL, + _("Failed to merge submodule %s (no merge base)"), + path); + goto cleanup; } if (!(commit_o = lookup_commit_reference(&subrepo, o)) || @@ -1771,6 +1797,7 @@ static int merge_submodule(struct merge_options *opt, path, NULL, NULL, NULL, _("Failed to merge submodule %s (commits not present)"), path); + sub_flag = CONFLICT_SUBMODULE_HISTORY_NOT_AVAILABLE; goto cleanup; } @@ -1849,7 +1876,23 @@ static int merge_submodule(struct merge_options *opt, object_array_clear(&merges); cleanup: - repo_clear(&subrepo); + if (!opt->priv->call_depth && !ret) { + struct string_list *csub = &opt->priv->conflicted_submodules; + struct conflicted_submodule_item *util; + const char *abbrev; + + util = xmalloc(sizeof(*util)); + util->flag = sub_flag; + util->abbrev = NULL; + if (!sub_not_initialized) { + abbrev = repo_find_unique_abbrev(&subrepo, b, DEFAULT_ABBREV); + util->abbrev = xstrdup(abbrev); + } + string_list_append(csub, path)->util = util; + } + + if (!sub_not_initialized) + repo_clear(&subrepo); return ret; } @@ -4377,22 +4420,8 @@ static int record_conflicted_index_entries(struct merge_options *opt) * the CE_SKIP_WORKTREE bit and manually write those * files to the working disk here. */ - if (ce_skip_worktree(ce)) { - struct stat st; - - if (!lstat(path, &st)) { - char *new_name = unique_path(opt, - path, - "cruft"); - - path_msg(opt, CONFLICT_PRESENT_DESPITE_SKIPPED, 1, - path, NULL, NULL, NULL, - _("Note: %s not up to date and in way of checking out conflicted version; old copy renamed to %s"), - path, new_name); - errs |= rename(path, new_name); - } + if (ce_skip_worktree(ce)) errs |= checkout_entry(ce, &state, NULL, NULL); - } /* * Mark this cache entry for removal and instead add @@ -4434,6 +4463,63 @@ static int record_conflicted_index_entries(struct merge_options *opt) return errs; } +static void print_submodule_conflict_suggestion(struct string_list *csub) { + struct string_list_item *item; + struct strbuf msg = STRBUF_INIT; + struct strbuf tmp = STRBUF_INIT; + struct strbuf subs = STRBUF_INIT; + + if (!csub->nr) + return; + + strbuf_add_separated_string_list(&subs, " ", csub); + for_each_string_list_item(item, csub) { + struct conflicted_submodule_item *util = item->util; + + /* + * NEEDSWORK: The steps to resolve these errors deserve a more + * detailed explanation than what is currently printed below. + */ + if (util->flag == CONFLICT_SUBMODULE_NOT_INITIALIZED || + util->flag == CONFLICT_SUBMODULE_HISTORY_NOT_AVAILABLE) + continue; + + /* + * TRANSLATORS: This is a line of advice to resolve a merge + * conflict in a submodule. The first argument is the submodule + * name, and the second argument is the abbreviated id of the + * commit that needs to be merged. For example: + * - go to submodule (mysubmodule), and either merge commit abc1234" + */ + strbuf_addf(&tmp, _(" - go to submodule (%s), and either merge commit %s\n" + " or update to an existing commit which has merged those changes\n"), + item->string, util->abbrev); + } + + /* + * TRANSLATORS: This is a detailed message for resolving submodule + * conflicts. The first argument is string containing one step per + * submodule. The second is a space-separated list of submodule names. + */ + strbuf_addf(&msg, + _("Recursive merging with submodules currently only supports trivial cases.\n" + "Please manually handle the merging of each conflicted submodule.\n" + "This can be accomplished with the following steps:\n" + "%s" + " - come back to superproject and run:\n\n" + " git add %s\n\n" + " to record the above merge or update\n" + " - resolve any other conflicts in the superproject\n" + " - commit the resulting index in the superproject\n"), + tmp.buf, subs.buf); + + printf("%s", msg.buf); + + strbuf_release(&subs); + strbuf_release(&tmp); + strbuf_release(&msg); +} + void merge_display_update_messages(struct merge_options *opt, int detailed, struct merge_result *result) @@ -4483,6 +4569,8 @@ void merge_display_update_messages(struct merge_options *opt, } string_list_clear(&olist, 0); + print_submodule_conflict_suggestion(&opti->conflicted_submodules); + /* Also include needed rename limit adjustment now */ diff_warn_rename_limit("merge.renamelimit", opti->renames.needed_limit, 0); @@ -4536,6 +4624,8 @@ void merge_switch_to_result(struct merge_options *opt, if (checkout(opt, head, result->tree)) { /* failure to function */ result->clean = -1; + merge_finalize(opt, result); + trace2_region_leave("merge", "checkout", opt->repo); return; } trace2_region_leave("merge", "checkout", opt->repo); @@ -4546,6 +4636,9 @@ void merge_switch_to_result(struct merge_options *opt, /* failure to function */ opt->priv = NULL; result->clean = -1; + merge_finalize(opt, result); + trace2_region_leave("merge", "record_conflicted", + opt->repo); return; } opt->priv = NULL; @@ -4679,6 +4772,7 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) trace2_region_enter("merge", "allocate/init", opt->repo); if (opt->priv) { clear_or_reinit_internal_opts(opt->priv, 1); + string_list_init_nodup(&opt->priv->conflicted_submodules); trace2_region_leave("merge", "allocate/init", opt->repo); return; } diff --git a/mergesort.c b/mergesort.c deleted file mode 100644 index bd9c6ef8ee..0000000000 --- a/mergesort.c +++ /dev/null @@ -1,84 +0,0 @@ -#include "cache.h" -#include "mergesort.h" - -/* Combine two sorted lists. Take from `list` on equality. */ -static void *llist_merge(void *list, void *other, - void *(*get_next_fn)(const void *), - void (*set_next_fn)(void *, void *), - int (*compare_fn)(const void *, const void *)) -{ - void *result = list, *tail; - - if (compare_fn(list, other) > 0) { - result = other; - goto other; - } - for (;;) { - do { - tail = list; - list = get_next_fn(list); - if (!list) { - set_next_fn(tail, other); - return result; - } - } while (compare_fn(list, other) <= 0); - set_next_fn(tail, other); - other: - do { - tail = other; - other = get_next_fn(other); - if (!other) { - set_next_fn(tail, list); - return result; - } - } while (compare_fn(list, other) > 0); - set_next_fn(tail, list); - } -} - -/* - * Perform an iterative mergesort using an array of sublists. - * - * n is the number of items. - * ranks[i] is undefined if n & 2^i == 0, and assumed empty. - * ranks[i] contains a sublist of length 2^i otherwise. - * - * The number of bits in a void pointer limits the number of objects - * that can be created, and thus the number of array elements necessary - * to be able to sort any valid list. - * - * Adding an item to this array is like incrementing a binary number; - * positional values for set bits correspond to sublist lengths. - */ -void *llist_mergesort(void *list, - void *(*get_next_fn)(const void *), - void (*set_next_fn)(void *, void *), - int (*compare_fn)(const void *, const void *)) -{ - void *ranks[bitsizeof(void *)]; - size_t n = 0; - int i; - - while (list) { - void *next = get_next_fn(list); - if (next) - set_next_fn(list, NULL); - for (i = 0; n & ((size_t)1 << i); i++) - list = llist_merge(ranks[i], list, get_next_fn, - set_next_fn, compare_fn); - n++; - ranks[i] = list; - list = next; - } - - for (i = 0; n; i++, n >>= 1) { - if (!(n & 1)) - continue; - if (list) - list = llist_merge(ranks[i], list, get_next_fn, - set_next_fn, compare_fn); - else - list = ranks[i]; - } - return list; -} diff --git a/mergesort.h b/mergesort.h index 644cff1f96..7c36f08bd5 100644 --- a/mergesort.h +++ b/mergesort.h @@ -1,17 +1,105 @@ #ifndef MERGESORT_H #define MERGESORT_H +/* Combine two sorted lists. Take from `list` on equality. */ +#define DEFINE_LIST_MERGE_INTERNAL(name, type) \ +static type *name##__merge(type *list, type *other, \ + int (*compare_fn)(const type *, const type *))\ +{ \ + type *result = list, *tail; \ + int prefer_list = compare_fn(list, other) <= 0; \ + \ + if (!prefer_list) { \ + result = other; \ + SWAP(list, other); \ + } \ + for (;;) { \ + do { \ + tail = list; \ + list = name##__get_next(list); \ + if (!list) { \ + name##__set_next(tail, other); \ + return result; \ + } \ + } while (compare_fn(list, other) < prefer_list); \ + name##__set_next(tail, other); \ + prefer_list ^= 1; \ + SWAP(list, other); \ + } \ +} + /* - * Sort linked list in place. - * - get_next_fn() returns the next element given an element of a linked list. - * - set_next_fn() takes two elements A and B, and makes B the "next" element - * of A on the list. - * - compare_fn() takes two elements A and B, and returns negative, 0, positive - * as the same sign as "subtracting" B from A. + * Perform an iterative mergesort using an array of sublists. + * + * n is the number of items. + * ranks[i] is undefined if n & 2^i == 0, and assumed empty. + * ranks[i] contains a sublist of length 2^i otherwise. + * + * The number of bits in a void pointer limits the number of objects + * that can be created, and thus the number of array elements necessary + * to be able to sort any valid list. + * + * Adding an item to this array is like incrementing a binary number; + * positional values for set bits correspond to sublist lengths. */ -void *llist_mergesort(void *list, - void *(*get_next_fn)(const void *), - void (*set_next_fn)(void *, void *), - int (*compare_fn)(const void *, const void *)); +#define DEFINE_LIST_SORT_INTERNAL(scope, name, type) \ +scope void name(type **listp, \ + int (*compare_fn)(const type *, const type *)) \ +{ \ + type *list = *listp; \ + type *ranks[bitsizeof(type *)]; \ + size_t n = 0; \ + \ + if (!list) \ + return; \ + \ + for (;;) { \ + int i; \ + size_t m; \ + type *next = name##__get_next(list); \ + if (next) \ + name##__set_next(list, NULL); \ + for (i = 0, m = n;; i++, m >>= 1) { \ + if (m & 1) { \ + list = name##__merge(ranks[i], list, \ + compare_fn); \ + } else if (next) { \ + break; \ + } else if (!m) { \ + *listp = list; \ + return; \ + } \ + } \ + n++; \ + ranks[i] = list; \ + list = next; \ + } \ +} + +#define DECLARE_LIST_SORT(scope, name, type) \ +scope void name(type **listp, \ + int (*compare_fn)(const type *, const type *)) + +#define DEFINE_LIST_SORT_DEBUG(scope, name, type, next_member, \ + on_get_next, on_set_next) \ + \ +static inline type *name##__get_next(const type *elem) \ +{ \ + on_get_next; \ + return elem->next_member; \ +} \ + \ +static inline void name##__set_next(type *elem, type *next) \ +{ \ + on_set_next; \ + elem->next_member = next; \ +} \ + \ +DEFINE_LIST_MERGE_INTERNAL(name, type) \ +DEFINE_LIST_SORT_INTERNAL(scope, name, type) \ +DECLARE_LIST_SORT(scope, name, type) + +#define DEFINE_LIST_SORT(scope, name, type, next_member) \ +DEFINE_LIST_SORT_DEBUG(scope, name, type, next_member, (void)0, (void)0) #endif diff --git a/mergetools/vimdiff b/mergetools/vimdiff index f770b8fe24..06937acbf5 100644 --- a/mergetools/vimdiff +++ b/mergetools/vimdiff @@ -29,8 +29,8 @@ ################################################################################ debug_print () { - # Send message to stderr if global variable GIT_MERGETOOL_VIMDIFF is set - # to "true" + # Send message to stderr if global variable GIT_MERGETOOL_VIMDIFF_DEBUG + # is set. if test -n "$GIT_MERGETOOL_VIMDIFF_DEBUG" then @@ -66,11 +66,6 @@ gen_cmd_aux () { debug_print "LAYOUT : $LAYOUT" debug_print "CMD : $CMD" - if test -z "$CMD" - then - CMD="echo" # vim "nop" operator - fi - start=0 end=${#LAYOUT} @@ -144,11 +139,10 @@ gen_cmd_aux () { # Step 2: # - # Search for all valid separators ("+", "/" or ",") which are *not* + # Search for all valid separators ("/" or ",") which are *not* # inside parenthesis. Save the index at which each of them makes the # first appearance. - index_new_tab="" index_horizontal_split="" index_vertical_split="" @@ -182,14 +176,7 @@ gen_cmd_aux () { then current=$c - if test "$current" = "+" - then - if test -z "$index_new_tab" - then - index_new_tab=$i - fi - - elif test "$current" = "/" + if test "$current" = "/" then if test -z "$index_horizontal_split" then @@ -219,14 +206,7 @@ gen_cmd_aux () { terminate="false" - if ! test -z "$index_new_tab" - then - before="-tabnew" - after="tabnext" - index=$index_new_tab - terminate="true" - - elif ! test -z "$index_horizontal_split" + if ! test -z "$index_horizontal_split" then before="leftabove split" after="wincmd j" @@ -333,25 +313,31 @@ gen_cmd () { # Obtain the first part of vim "-c" option to obtain the desired layout - CMD=$(gen_cmd_aux "$LAYOUT") - - - # Adjust the just obtained script depending on whether more than one - # windows are visible or not + CMD= + oldIFS=$IFS + IFS=+ + for tab in $LAYOUT + do + if test -z "$CMD" + then + CMD="echo" # vim "nop" operator + else + CMD="$CMD | tabnew" + fi - if echo "$LAYOUT" | grep ",\|/" >/dev/null - then - CMD="$CMD | tabdo windo diffthis" - else - CMD="$CMD | bufdo diffthis" - fi + # If this is a single window diff with all the buffers + if ! echo "$tab" | grep ",\|/" >/dev/null + then + CMD="$CMD | silent execute 'bufdo diffthis'" + fi + CMD=$(gen_cmd_aux "$tab" "$CMD") + done + IFS=$oldIFS - # Add an extra "-c" option to move to the first tab (notice that we - # can't simply append the command to the previous "-c" string as - # explained here: https://github.com/vim/vim/issues/9076 + CMD="$CMD | execute 'tabdo windo diffthis'" - FINAL_CMD="-c \"$CMD\" -c \"tabfirst\"" + FINAL_CMD="-c \"set hidden diffopt-=hiddenoff | $CMD | tabfirst\"" } @@ -555,22 +541,22 @@ run_unit_tests () { TEST_CASE_15=" (( (LOCAL , BASE , REMOTE) / MERGED)) +(BASE) , LOCAL+ BASE , REMOTE+ (((LOCAL / BASE / REMOTE)) , MERGED ) " TEST_CASE_16="LOCAL,BASE,REMOTE / MERGED + BASE,LOCAL + BASE,REMOTE + (LOCAL / BASE / REMOTE),MERGED" - EXPECTED_CMD_01="-c \"echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_02="-c \"echo | leftabove vertical split | 1b | wincmd l | 3b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_03="-c \"echo | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 4b | wincmd l | 3b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_04="-c \"echo | 4b | bufdo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_05="-c \"echo | leftabove split | 1b | wincmd j | leftabove split | 4b | wincmd j | 3b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_06="-c \"echo | leftabove vertical split | leftabove split | 1b | wincmd j | 3b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_07="-c \"echo | leftabove vertical split | 4b | wincmd l | leftabove split | 1b | wincmd j | 3b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_08="-c \"echo | leftabove split | leftabove vertical split | 1b | wincmd l | 3b | wincmd j | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_09="-c \"echo | leftabove split | 4b | wincmd j | leftabove vertical split | 1b | wincmd l | 3b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_10="-c \"echo | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_11="-c \"echo | -tabnew | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabnext | -tabnew | leftabove vertical split | 2b | wincmd l | 1b | tabnext | -tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnext | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_12="-c \"echo | leftabove vertical split | leftabove split | leftabove vertical split | 1b | wincmd l | 3b | wincmd j | 2b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_13="-c \"echo | leftabove vertical split | leftabove split | leftabove vertical split | 1b | wincmd l | 3b | wincmd j | 2b | wincmd l | leftabove vertical split | leftabove split | 1b | wincmd j | 3b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_14="-c \"echo | -tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnext | leftabove vertical split | 2b | wincmd l | 1b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_15="-c \"echo | -tabnew | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabnext | -tabnew | leftabove vertical split | 2b | wincmd l | 1b | tabnext | -tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnext | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" - EXPECTED_CMD_16="-c \"echo | -tabnew | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabnext | -tabnew | leftabove vertical split | 2b | wincmd l | 1b | tabnext | -tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnext | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | tabdo windo diffthis\" -c \"tabfirst\"" + EXPECTED_CMD_01="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_02="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | 1b | wincmd l | 3b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_03="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 4b | wincmd l | 3b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_04="-c \"set hidden diffopt-=hiddenoff | echo | silent execute 'bufdo diffthis' | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_05="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | 1b | wincmd j | leftabove split | 4b | wincmd j | 3b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_06="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | leftabove split | 1b | wincmd j | 3b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_07="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | 4b | wincmd l | leftabove split | 1b | wincmd j | 3b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_08="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | leftabove vertical split | 1b | wincmd l | 3b | wincmd j | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_09="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | 4b | wincmd j | leftabove vertical split | 1b | wincmd l | 3b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_10="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_11="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabnew | leftabove vertical split | 2b | wincmd l | 1b | tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnew | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_12="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | leftabove split | leftabove vertical split | 1b | wincmd l | 3b | wincmd j | 2b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_13="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | leftabove split | leftabove vertical split | 1b | wincmd l | 3b | wincmd j | 2b | wincmd l | leftabove vertical split | leftabove split | 1b | wincmd j | 3b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_14="-c \"set hidden diffopt-=hiddenoff | echo | leftabove vertical split | 2b | wincmd l | 3b | tabnew | leftabove vertical split | 2b | wincmd l | 1b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_15="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabnew | leftabove vertical split | 2b | wincmd l | 1b | tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnew | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" + EXPECTED_CMD_16="-c \"set hidden diffopt-=hiddenoff | echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | 2b | wincmd l | 3b | wincmd j | 4b | tabnew | leftabove vertical split | 2b | wincmd l | 1b | tabnew | leftabove vertical split | 2b | wincmd l | 3b | tabnew | leftabove vertical split | leftabove split | 1b | wincmd j | leftabove split | 2b | wincmd j | 3b | wincmd l | 4b | execute 'tabdo windo diffthis' | tabfirst\"" EXPECTED_TARGET_01="MERGED" EXPECTED_TARGET_02="LOCAL" @@ -635,9 +621,7 @@ run_unit_tests () { cat >expect <<-\EOF -f -c - echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | quit | wincmd l | 2b | wincmd j | 3b | tabdo windo diffthis - -c - tabfirst + set hidden diffopt-=hiddenoff | echo | leftabove split | leftabove vertical split | 1b | wincmd l | leftabove vertical split | quit | wincmd l | 2b | wincmd j | 3b | execute 'tabdo windo diffthis' | tabfirst lo cal ' ' mer ged @@ -1053,40 +1053,34 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr return cb.commits; } -static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash, - struct write_midx_context *ctx, - const char *refs_snapshot, +static int write_midx_bitmap(const char *midx_name, + const unsigned char *midx_hash, + struct packing_data *pdata, + struct commit **commits, + uint32_t commits_nr, + uint32_t *pack_order, unsigned flags) { - struct packing_data pdata; - struct pack_idx_entry **index; - struct commit **commits = NULL; - uint32_t i, commits_nr; + int ret, i; uint16_t options = 0; - char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, hash_to_hex(midx_hash)); - int ret; - - if (!ctx->entries_nr) - BUG("cannot write a bitmap without any objects"); + struct pack_idx_entry **index; + char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, + hash_to_hex(midx_hash)); if (flags & MIDX_WRITE_BITMAP_HASH_CACHE) options |= BITMAP_OPT_HASH_CACHE; - prepare_midx_packing_data(&pdata, ctx); - - commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, ctx); - /* * Build the MIDX-order index based on pdata.objects (which is already * in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of * this order). */ - ALLOC_ARRAY(index, pdata.nr_objects); - for (i = 0; i < pdata.nr_objects; i++) - index[i] = &pdata.objects[i].idx; + ALLOC_ARRAY(index, pdata->nr_objects); + for (i = 0; i < pdata->nr_objects; i++) + index[i] = &pdata->objects[i].idx; bitmap_writer_show_progress(flags & MIDX_PROGRESS); - bitmap_writer_build_type_index(&pdata, index, pdata.nr_objects); + bitmap_writer_build_type_index(pdata, index, pdata->nr_objects); /* * bitmap_writer_finish expects objects in lex order, but pack_order @@ -1101,16 +1095,16 @@ static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash, * happens between bitmap_writer_build_type_index() and * bitmap_writer_finish(). */ - for (i = 0; i < pdata.nr_objects; i++) - index[ctx->pack_order[i]] = &pdata.objects[i].idx; + for (i = 0; i < pdata->nr_objects; i++) + index[pack_order[i]] = &pdata->objects[i].idx; bitmap_writer_select_commits(commits, commits_nr, -1); - ret = bitmap_writer_build(&pdata); + ret = bitmap_writer_build(pdata); if (ret < 0) goto cleanup; bitmap_writer_set_checksum(midx_hash); - bitmap_writer_finish(index, pdata.nr_objects, bitmap_name, options); + bitmap_writer_finish(index, pdata->nr_objects, bitmap_name, options); cleanup: free(index); @@ -1443,14 +1437,40 @@ static int write_midx_internal(const char *object_dir, if (flags & MIDX_WRITE_REV_INDEX && git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) write_midx_reverse_index(midx_name.buf, midx_hash, &ctx); + if (flags & MIDX_WRITE_BITMAP) { - if (write_midx_bitmap(midx_name.buf, midx_hash, &ctx, - refs_snapshot, flags) < 0) { + struct packing_data pdata; + struct commit **commits; + uint32_t commits_nr; + + if (!ctx.entries_nr) + BUG("cannot write a bitmap without any objects"); + + prepare_midx_packing_data(&pdata, &ctx); + + commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, &ctx); + + /* + * The previous steps translated the information from + * 'entries' into information suitable for constructing + * bitmaps. We no longer need that array, so clear it to + * reduce memory pressure. + */ + FREE_AND_NULL(ctx.entries); + ctx.entries_nr = 0; + + if (write_midx_bitmap(midx_name.buf, midx_hash, &pdata, + commits, commits_nr, ctx.pack_order, + flags) < 0) { error(_("could not write multi-pack bitmap")); result = 1; goto cleanup; } } + /* + * NOTE: Do not use ctx.entries beyond this point, since it might + * have been freed in the previous if block. + */ if (ctx.m) close_object_store(the_repository->objects); @@ -1005,6 +1005,7 @@ void init_notes(struct notes_tree *t, const char *notes_ref, if (!notes_ref) notes_ref = default_notes_ref(); + update_ref_namespace(NAMESPACE_NOTES, xstrdup(notes_ref)); if (!combine_notes) combine_notes = combine_notes_concatenate; @@ -59,7 +59,7 @@ struct object_array { /* * object flag allocation: - * revision.h: 0---------10 15 23------26 + * revision.h: 0---------10 15 23------27 * fetch-pack.c: 01 67 * negotiator/default.c: 2--5 * walker.c: 0-2 diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index c43375bd34..4fcfaed428 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -683,7 +683,7 @@ static void write_hash_cache(struct hashfile *f, } } -void bitmap_writer_set_checksum(unsigned char *sha1) +void bitmap_writer_set_checksum(const unsigned char *sha1) { hashcpy(writer.pack_checksum, sha1); } diff --git a/pack-bitmap.c b/pack-bitmap.c index 36134222d7..ef580be9e3 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -1,5 +1,6 @@ #include "cache.h" #include "commit.h" +#include "strbuf.h" #include "tag.h" #include "diff.h" #include "revision.h" @@ -138,7 +139,7 @@ static struct ewah_bitmap *read_bitmap_1(struct bitmap_index *index) index->map_size - index->map_pos); if (bitmap_size < 0) { - error("Failed to load bitmap index (corrupted?)"); + error(_("failed to load bitmap index (corrupted?)")); ewah_pool_free(b); return NULL; } @@ -160,14 +161,14 @@ static int load_bitmap_header(struct bitmap_index *index) size_t header_size = sizeof(*header) - GIT_MAX_RAWSZ + the_hash_algo->rawsz; if (index->map_size < header_size + the_hash_algo->rawsz) - return error("Corrupted bitmap index (too small)"); + return error(_("corrupted bitmap index (too small)")); if (memcmp(header->magic, BITMAP_IDX_SIGNATURE, sizeof(BITMAP_IDX_SIGNATURE)) != 0) - return error("Corrupted bitmap index file (wrong header)"); + return error(_("corrupted bitmap index file (wrong header)")); index->version = ntohs(header->version); if (index->version != 1) - return error("Unsupported version for bitmap index file (%d)", index->version); + return error(_("unsupported version '%d' for bitmap index file"), index->version); /* Parse known bitmap format options */ { @@ -176,12 +177,12 @@ static int load_bitmap_header(struct bitmap_index *index) unsigned char *index_end = index->map + index->map_size - the_hash_algo->rawsz; if ((flags & BITMAP_OPT_FULL_DAG) == 0) - return error("Unsupported options for bitmap index file " + BUG("unsupported options for bitmap index file " "(Git requires BITMAP_OPT_FULL_DAG)"); if (flags & BITMAP_OPT_HASH_CACHE) { if (cache_size > index_end - index->map - header_size) - return error("corrupted bitmap index file (too short to fit hash cache)"); + return error(_("corrupted bitmap index file (too short to fit hash cache)")); index->hashes = (void *)(index_end - cache_size); index_end -= cache_size; } @@ -215,7 +216,7 @@ static struct stored_bitmap *store_bitmap(struct bitmap_index *index, * because the SHA1 already existed on the map. this is bad, there * shouldn't be duplicated commits in the index */ if (ret == 0) { - error("Duplicate entry in bitmap index: %s", oid_to_hex(oid)); + error(_("duplicate entry in bitmap index: '%s'"), oid_to_hex(oid)); return NULL; } @@ -259,14 +260,14 @@ static int load_bitmap_entries_v1(struct bitmap_index *index) struct object_id oid; if (index->map_size - index->map_pos < 6) - return error("corrupt ewah bitmap: truncated header for entry %d", i); + return error(_("corrupt ewah bitmap: truncated header for entry %d"), i); commit_idx_pos = read_be32(index->map, &index->map_pos); xor_offset = read_u8(index->map, &index->map_pos); flags = read_u8(index->map, &index->map_pos); if (nth_bitmap_object_oid(index, &oid, commit_idx_pos) < 0) - return error("corrupt ewah bitmap: commit index %u out of range", + return error(_("corrupt ewah bitmap: commit index %u out of range"), (unsigned)commit_idx_pos); bitmap = read_bitmap_1(index); @@ -274,13 +275,13 @@ static int load_bitmap_entries_v1(struct bitmap_index *index) return -1; if (xor_offset > MAX_XOR_OFFSET || xor_offset > i) - return error("Corrupted bitmap pack index"); + return error(_("corrupted bitmap pack index")); if (xor_offset > 0) { xor_bitmap = recent_bitmaps[(i - xor_offset) % MAX_XOR_OFFSET]; if (!xor_bitmap) - return error("Invalid XOR offset in bitmap pack index"); + return error(_("invalid XOR offset in bitmap pack index")); } recent_bitmaps[i % MAX_XOR_OFFSET] = store_bitmap( @@ -313,17 +314,21 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, struct multi_pack_index *midx) { struct stat st; - char *idx_name = midx_bitmap_filename(midx); - int fd = git_open(idx_name); + char *bitmap_name = midx_bitmap_filename(midx); + int fd = git_open(bitmap_name); uint32_t i; struct packed_git *preferred; - free(idx_name); - - if (fd < 0) + if (fd < 0) { + if (errno != ENOENT) + warning_errno("cannot open '%s'", bitmap_name); + free(bitmap_name); return -1; + } + free(bitmap_name); if (fstat(fd, &st)) { + error_errno(_("cannot fstat bitmap file")); close(fd); return -1; } @@ -332,7 +337,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, struct strbuf buf = STRBUF_INIT; get_midx_filename(&buf, midx->object_dir); /* ignore extra bitmap file; we can only handle one */ - warning("ignoring extra bitmap file: %s", buf.buf); + warning(_("ignoring extra bitmap file: '%s'"), buf.buf); close(fd); strbuf_release(&buf); return -1; @@ -348,8 +353,10 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, if (load_bitmap_header(bitmap_git) < 0) goto cleanup; - if (!hasheq(get_midx_checksum(bitmap_git->midx), bitmap_git->checksum)) + if (!hasheq(get_midx_checksum(bitmap_git->midx), bitmap_git->checksum)) { + error(_("checksum doesn't match in MIDX and bitmap")); goto cleanup; + } if (load_midx_revindex(bitmap_git->midx) < 0) { warning(_("multi-pack bitmap is missing required reverse index")); @@ -384,26 +391,31 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git { int fd; struct stat st; - char *idx_name; + char *bitmap_name; if (open_pack_index(packfile)) return -1; - idx_name = pack_bitmap_filename(packfile); - fd = git_open(idx_name); - free(idx_name); + bitmap_name = pack_bitmap_filename(packfile); + fd = git_open(bitmap_name); - if (fd < 0) + if (fd < 0) { + if (errno != ENOENT) + warning_errno("cannot open '%s'", bitmap_name); + free(bitmap_name); return -1; + } + free(bitmap_name); if (fstat(fd, &st)) { + error_errno(_("cannot fstat bitmap file")); close(fd); return -1; } if (bitmap_git->pack || bitmap_git->midx) { /* ignore extra bitmap file; we can only handle one */ - warning("ignoring extra bitmap file: %s", packfile->pack_name); + warning(_("ignoring extra bitmap file: '%s'"), packfile->pack_name); close(fd); return -1; } @@ -508,15 +520,16 @@ static int open_pack_bitmap(struct repository *r, static int open_midx_bitmap(struct repository *r, struct bitmap_index *bitmap_git) { + int ret = -1; struct multi_pack_index *midx; assert(!bitmap_git->map); for (midx = get_multi_pack_index(r); midx; midx = midx->next) { if (!open_midx_bitmap_1(bitmap_git, midx)) - return 0; + ret = 0; } - return -1; + return ret; } static int open_bitmap(struct repository *r, @@ -831,7 +844,7 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git, revs->include_check_data = &incdata; if (prepare_revision_walk(revs)) - die("revision walk setup failed"); + die(_("revision walk setup failed")); show_data.bitmap_git = bitmap_git; show_data.base = base; @@ -1640,15 +1653,15 @@ static void test_bitmap_type(struct bitmap_test_data *tdata, } if (bitmap_type == OBJ_NONE) - die("object %s not found in type bitmaps", + die(_("object '%s' not found in type bitmaps"), oid_to_hex(&obj->oid)); if (bitmaps_nr > 1) - die("object %s does not have a unique type", + die(_("object '%s' does not have a unique type"), oid_to_hex(&obj->oid)); if (bitmap_type != obj->type) - die("object %s: real type %s, expected: %s", + die(_("object '%s': real type '%s', expected: '%s'"), oid_to_hex(&obj->oid), type_name(obj->type), type_name(bitmap_type)); @@ -1662,7 +1675,7 @@ static void test_show_object(struct object *object, const char *name, bitmap_pos = bitmap_position(tdata->bitmap_git, &object->oid); if (bitmap_pos < 0) - die("Object not in bitmap: %s\n", oid_to_hex(&object->oid)); + die(_("object not in bitmap: '%s'"), oid_to_hex(&object->oid)); test_bitmap_type(tdata, object, bitmap_pos); bitmap_set(tdata->base, bitmap_pos); @@ -1677,7 +1690,7 @@ static void test_show_commit(struct commit *commit, void *data) bitmap_pos = bitmap_position(tdata->bitmap_git, &commit->object.oid); if (bitmap_pos < 0) - die("Object not in bitmap: %s\n", oid_to_hex(&commit->object.oid)); + die(_("object not in bitmap: '%s'"), oid_to_hex(&commit->object.oid)); test_bitmap_type(tdata, &commit->object, bitmap_pos); bitmap_set(tdata->base, bitmap_pos); @@ -1694,26 +1707,26 @@ void test_bitmap_walk(struct rev_info *revs) struct ewah_bitmap *bm; if (!(bitmap_git = prepare_bitmap_git(revs->repo))) - die("failed to load bitmap indexes"); + die(_("failed to load bitmap indexes")); if (revs->pending.nr != 1) - die("you must specify exactly one commit to test"); + die(_("you must specify exactly one commit to test")); - fprintf(stderr, "Bitmap v%d test (%d entries loaded)\n", + fprintf_ln(stderr, "Bitmap v%d test (%d entries loaded)", bitmap_git->version, bitmap_git->entry_count); root = revs->pending.objects[0].item; bm = bitmap_for_commit(bitmap_git, (struct commit *)root); if (bm) { - fprintf(stderr, "Found bitmap for %s. %d bits / %08x checksum\n", + fprintf_ln(stderr, "Found bitmap for '%s'. %d bits / %08x checksum", oid_to_hex(&root->oid), (int)bm->bit_size, ewah_checksum(bm)); result = ewah_to_bitmap(bm); } if (!result) - die("Commit %s doesn't have an indexed bitmap", oid_to_hex(&root->oid)); + die(_("commit '%s' doesn't have an indexed bitmap"), oid_to_hex(&root->oid)); revs->tag_objects = 1; revs->tree_objects = 1; @@ -1722,7 +1735,7 @@ void test_bitmap_walk(struct rev_info *revs) result_popcnt = bitmap_popcount(result); if (prepare_revision_walk(revs)) - die("revision walk setup failed"); + die(_("revision walk setup failed")); tdata.bitmap_git = bitmap_git; tdata.base = bitmap_new(); @@ -1738,9 +1751,9 @@ void test_bitmap_walk(struct rev_info *revs) stop_progress(&tdata.prg); if (bitmap_equals(result, tdata.base)) - fprintf(stderr, "OK!\n"); + fprintf_ln(stderr, "OK!"); else - die("mismatch in bitmap results"); + die(_("mismatch in bitmap results")); bitmap_free(result); bitmap_free(tdata.base); @@ -1758,10 +1771,10 @@ int test_bitmap_commits(struct repository *r) MAYBE_UNUSED void *value; if (!bitmap_git) - die("failed to load bitmap indexes"); + die(_("failed to load bitmap indexes")); kh_foreach(bitmap_git->bitmaps, oid, value, { - printf("%s\n", oid_to_hex(&oid)); + printf_ln("%s", oid_to_hex(&oid)); }); free_bitmap_index(bitmap_git); @@ -1786,7 +1799,7 @@ int test_bitmap_hashes(struct repository *r) nth_bitmap_object_oid(bitmap_git, &oid, index_pos); - printf("%s %"PRIu32"\n", + printf_ln("%s %"PRIu32"", oid_to_hex(&oid), get_be32(bitmap_git->hashes + index_pos)); } @@ -1948,7 +1961,7 @@ static off_t get_disk_usage_for_type(struct bitmap_index *bitmap_git, struct object_id oid; nth_midxed_object_oid(&oid, bitmap_git->midx, midx_pos); - die(_("could not find %s in pack %s at offset %"PRIuMAX), + die(_("could not find '%s' in pack '%s' at offset %"PRIuMAX), oid_to_hex(&oid), pack->pack_name, (uintmax_t)offset); @@ -1984,7 +1997,7 @@ static off_t get_disk_usage_for_extended(struct bitmap_index *bitmap_git) continue; if (oid_object_info_extended(the_repository, &obj->oid, &oi, 0) < 0) - die(_("unable to get disk usage of %s"), + die(_("unable to get disk usage of '%s'"), oid_to_hex(&obj->oid)); total += object_size; diff --git a/pack-bitmap.h b/pack-bitmap.h index 3d3ddd7734..f3a57ca065 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -75,7 +75,7 @@ int bitmap_has_oid_in_uninteresting(struct bitmap_index *, const struct object_i off_t get_disk_usage_from_bitmap(struct bitmap_index *, struct rev_info *); void bitmap_writer_show_progress(int show); -void bitmap_writer_set_checksum(unsigned char *sha1); +void bitmap_writer_set_checksum(const unsigned char *sha1); void bitmap_writer_build_type_index(struct packing_data *to_pack, struct pack_idx_entry **index, uint32_t index_nr); diff --git a/pack-revindex.h b/pack-revindex.h index 74f4eae668..4974e75eb4 100644 --- a/pack-revindex.h +++ b/pack-revindex.h @@ -22,7 +22,7 @@ * * - pack position refers to an object's position within a non-existent pack * described by the MIDX. The pack structure is described in - * Documentation/technical/pack-format.txt. + * gitformat-pack(5). * * It is effectively a concatanation of all packs in the MIDX (ordered by * their numeric ID within the MIDX) in their original order within each diff --git a/packfile.c b/packfile.c index ed69fe457b..a41887c944 100644 --- a/packfile.c +++ b/packfile.c @@ -941,20 +941,10 @@ unsigned long repo_approximate_object_count(struct repository *r) return r->objects->approximate_object_count; } -static void *get_next_packed_git(const void *p) -{ - return ((const struct packed_git *)p)->next; -} - -static void set_next_packed_git(void *p, void *next) -{ - ((struct packed_git *)p)->next = next; -} +DEFINE_LIST_SORT(static, sort_packs, struct packed_git, next); -static int sort_pack(const void *a_, const void *b_) +static int sort_pack(const struct packed_git *a, const struct packed_git *b) { - const struct packed_git *a = a_; - const struct packed_git *b = b_; int st; /* @@ -981,9 +971,7 @@ static int sort_pack(const void *a_, const void *b_) static void rearrange_packed_git(struct repository *r) { - r->objects->packed_git = llist_mergesort( - r->objects->packed_git, get_next_packed_git, - set_next_packed_git, sort_pack); + sort_packs(&r->objects->packed_git, sort_pack); } static void prepare_packed_git_mru(struct repository *r) @@ -2229,7 +2217,17 @@ static int add_promisor_object(const struct object_id *oid, void *set_) { struct oidset *set = set_; - struct object *obj = parse_object(the_repository, oid); + struct object *obj; + int we_parsed_object; + + obj = lookup_object(the_repository, oid); + if (obj && obj->parsed) { + we_parsed_object = 0; + } else { + we_parsed_object = 1; + obj = parse_object(the_repository, oid); + } + if (!obj) return 1; @@ -2243,7 +2241,7 @@ static int add_promisor_object(const struct object_id *oid, struct tree *tree = (struct tree *)obj; struct tree_desc desc; struct name_entry entry; - if (init_tree_desc_gently(&desc, tree->buffer, tree->size)) + if (init_tree_desc_gently(&desc, tree->buffer, tree->size, 0)) /* * Error messages are given when packs are * verified, so do not print any here. @@ -2251,7 +2249,8 @@ static int add_promisor_object(const struct object_id *oid, return 0; while (tree_entry_gently(&desc, &entry)) oidset_insert(set, &entry.oid); - free_tree_buffer(tree); + if (we_parsed_object) + free_tree_buffer(tree); } else if (obj->type == OBJ_COMMIT) { struct commit *commit = (struct commit *) obj; struct commit_list *parents = commit->parents; diff --git a/parallel-checkout.c b/parallel-checkout.c index 31a3d0ee1b..4f6819f240 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -143,7 +143,8 @@ static int is_eligible_for_parallel_checkout(const struct cache_entry *ce, } } -int enqueue_checkout(struct cache_entry *ce, struct conv_attrs *ca) +int enqueue_checkout(struct cache_entry *ce, struct conv_attrs *ca, + int *checkout_counter) { struct parallel_checkout_item *pc_item; @@ -159,6 +160,7 @@ int enqueue_checkout(struct cache_entry *ce, struct conv_attrs *ca) memcpy(&pc_item->ca, ca, sizeof(pc_item->ca)); pc_item->status = PC_ITEM_PENDING; pc_item->id = parallel_checkout.nr; + pc_item->checkout_counter = checkout_counter; parallel_checkout.nr++; return 0; @@ -200,7 +202,8 @@ static int handle_results(struct checkout *state) switch(pc_item->status) { case PC_ITEM_WRITTEN: - /* Already handled */ + if (pc_item->checkout_counter) + (*pc_item->checkout_counter)++; break; case PC_ITEM_COLLIDED: /* @@ -225,7 +228,8 @@ static int handle_results(struct checkout *state) * add any extra overhead. */ ret |= checkout_entry_ca(pc_item->ce, &pc_item->ca, - state, NULL, NULL); + state, NULL, + pc_item->checkout_counter); advance_progress_meter(); break; case PC_ITEM_PENDING: diff --git a/parallel-checkout.h b/parallel-checkout.h index 80f539bcb7..c575284005 100644 --- a/parallel-checkout.h +++ b/parallel-checkout.h @@ -31,7 +31,8 @@ void init_parallel_checkout(void); * entry is not eligible for parallel checkout. Otherwise, enqueue the entry * for later write and return 0. */ -int enqueue_checkout(struct cache_entry *ce, struct conv_attrs *ca); +int enqueue_checkout(struct cache_entry *ce, struct conv_attrs *ca, + int *checkout_counter); size_t pc_queue_size(void); /* @@ -68,6 +69,7 @@ struct parallel_checkout_item { struct cache_entry *ce; struct conv_attrs ca; size_t id; /* position in parallel_checkout.items[] of main process */ + int *checkout_counter; /* Output fields, sent from workers. */ enum pc_item_status status; diff --git a/pathspec.c b/pathspec.c index 84ad9c73cf..46e77a85fe 100644 --- a/pathspec.c +++ b/pathspec.c @@ -759,3 +759,92 @@ int match_pathspec_attrs(struct index_state *istate, return 1; } + +int pathspec_needs_expanded_index(struct index_state *istate, + const struct pathspec *pathspec) +{ + unsigned int i, pos; + int res = 0; + char *skip_worktree_seen = NULL; + + /* + * If index is not sparse, no index expansion is needed. + */ + if (!istate->sparse_index) + return 0; + + /* + * When using a magic pathspec, assume for the sake of simplicity that + * the index needs to be expanded to match all matchable files. + */ + if (pathspec->magic) + return 1; + + for (i = 0; i < pathspec->nr; i++) { + struct pathspec_item item = pathspec->items[i]; + + /* + * If the pathspec item has a wildcard, the index should be expanded + * if the pathspec has the possibility of matching a subset of entries inside + * of a sparse directory (but not the entire directory). + * + * If the pathspec item is a literal path, the index only needs to be expanded + * if a) the pathspec isn't in the sparse checkout cone (to make sure we don't + * expand for in-cone files) and b) it doesn't match any sparse directories + * (since we can reset whole sparse directories without expanding them). + */ + if (item.nowildcard_len < item.len) { + /* + * Special case: if the pattern is a path inside the cone + * followed by only wildcards, the pattern cannot match + * partial sparse directories, so we know we don't need to + * expand the index. + * + * Examples: + * - in-cone/foo***: doesn't need expanded index + * - not-in-cone/bar*: may need expanded index + * - **.c: may need expanded index + */ + if (strspn(item.original + item.nowildcard_len, "*") == item.len - item.nowildcard_len && + path_in_cone_mode_sparse_checkout(item.original, istate)) + continue; + + for (pos = 0; pos < istate->cache_nr; pos++) { + struct cache_entry *ce = istate->cache[pos]; + + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + + /* + * If the pre-wildcard length is longer than the sparse + * directory name and the sparse directory is the first + * component of the pathspec, need to expand the index. + */ + if (item.nowildcard_len > ce_namelen(ce) && + !strncmp(item.original, ce->name, ce_namelen(ce))) { + res = 1; + break; + } + + /* + * If the pre-wildcard length is shorter than the sparse + * directory and the pathspec does not match the whole + * directory, need to expand the index. + */ + if (!strncmp(item.original, ce->name, item.nowildcard_len) && + wildmatch(item.original, ce->name, 0)) { + res = 1; + break; + } + } + } else if (!path_in_cone_mode_sparse_checkout(item.original, istate) && + !matches_skip_worktree(pathspec, i, &skip_worktree_seen)) + res = 1; + + if (res > 0) + break; + } + + free(skip_worktree_seen); + return res; +} diff --git a/pathspec.h b/pathspec.h index 402ebb8080..41f6adfbb4 100644 --- a/pathspec.h +++ b/pathspec.h @@ -171,4 +171,16 @@ int match_pathspec_attrs(struct index_state *istate, const char *name, int namelen, const struct pathspec_item *item); +/* + * Determine whether a pathspec will match only entire index entries (non-sparse + * files and/or entire sparse directories). If the pathspec has the potential to + * match partial contents of a sparse directory, return 1 to indicate the index + * should be expanded to match the appropriate index entries. + * + * For the sake of simplicity, always return 1 if using a more complex "magic" + * pathspec. + */ +int pathspec_needs_expanded_index(struct index_state *istate, + const struct pathspec *pathspec); + #endif /* PATHSPEC_H */ diff --git a/pkt-line.c b/pkt-line.c index 8e43c2def4..ce4e73b683 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -309,7 +309,8 @@ int write_packetized_from_fd_no_flush(int fd_in, int fd_out) return err; } -int write_packetized_from_buf_no_flush(const char *src_in, size_t len, int fd_out) +int write_packetized_from_buf_no_flush_count(const char *src_in, size_t len, + int fd_out, int *packet_counter) { int err = 0; size_t bytes_written = 0; @@ -324,6 +325,8 @@ int write_packetized_from_buf_no_flush(const char *src_in, size_t len, int fd_ou break; err = packet_write_gently(fd_out, src_in + bytes_written, bytes_to_write); bytes_written += bytes_to_write; + if (packet_counter) + (*packet_counter)++; } return err; } diff --git a/pkt-line.h b/pkt-line.h index 6d2a63db23..79c538b99e 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -32,7 +32,13 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((f int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int write_packetized_from_fd_no_flush(int fd_in, int fd_out); -int write_packetized_from_buf_no_flush(const char *src_in, size_t len, int fd_out); +int write_packetized_from_buf_no_flush_count(const char *src_in, size_t len, + int fd_out, int *packet_counter); +static inline int write_packetized_from_buf_no_flush(const char *src_in, + size_t len, int fd_out) +{ + return write_packetized_from_buf_no_flush_count(src_in, len, fd_out, NULL); +} /* * Stdio versions of packet_write functions. When mixing these with fd @@ -49,14 +55,6 @@ void packet_fflush(FILE *f); * Read a packetized line into the buffer, which must be at least size bytes * long. The return value specifies the number of bytes read into the buffer. * - * If src_buffer and *src_buffer are not NULL, it should point to a buffer - * containing the packet data to parse, of at least *src_len bytes. After the - * function returns, src_buf will be incremented and src_len decremented by the - * number of bytes consumed. - * - * If src_buffer (or *src_buffer) is NULL, then data is read from the - * descriptor "fd". - * * If options does not contain PACKET_READ_GENTLE_ON_EOF, we will die under any * of the following conditions: * @@ -104,6 +102,14 @@ int packet_length(const char lenbuf_hex[4]); * returns an 'enum packet_read_status' which indicates the status of the read. * The number of bytes read will be assigned to *pktlen if the status of the * read was 'PACKET_READ_NORMAL'. + * + * If src_buffer and *src_buffer are not NULL, it should point to a buffer + * containing the packet data to parse, of at least *src_len bytes. After the + * function returns, src_buf will be incremented and src_len decremented by the + * number of bytes consumed. + * + * If src_buffer (or *src_buffer) is NULL, then data is read from the + * descriptor "fd". */ enum packet_read_status { PACKET_READ_EOF, diff --git a/preload-index.c b/preload-index.c index e5529a5863..100f7a374d 100644 --- a/preload-index.c +++ b/preload-index.c @@ -151,6 +151,12 @@ void preload_index(struct index_state *index, } stop_progress(&pd.progress); + if (pathspec) { + /* earlier we made deep copies for each thread to work with */ + for (i = 0; i < threads; i++) + clear_pathspec(&data[i].pathspec); + } + trace_performance_leave("preload index"); trace2_data_intmax("index", NULL, "preload/sum_lstat", t2_sum_lstat); diff --git a/promisor-remote.c b/promisor-remote.c index 5b33f88bca..68f46f5ec7 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -146,7 +146,7 @@ static void promisor_remote_init(struct repository *r) if (r->promisor_remote_config) return; config = r->promisor_remote_config = - xcalloc(sizeof(*r->promisor_remote_config), 1); + xcalloc(1, sizeof(*r->promisor_remote_config)); config->promisors_tail = &config->promisors; repo_config(r, promisor_remote_config, config); diff --git a/read-cache.c b/read-cache.c index 76f372ff91..b09128b188 100644 --- a/read-cache.c +++ b/read-cache.c @@ -620,6 +620,11 @@ int index_name_pos(struct index_state *istate, const char *name, int namelen) return index_name_stage_pos(istate, name, namelen, 0, EXPAND_SPARSE); } +int index_name_pos_sparse(struct index_state *istate, const char *name, int namelen) +{ + return index_name_stage_pos(istate, name, namelen, 0, NO_EXPAND_SPARSE); +} + int index_entry_exists(struct index_state *istate, const char *name, int namelen) { return index_name_stage_pos(istate, name, namelen, 0, NO_EXPAND_SPARSE) >= 0; @@ -2294,6 +2299,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) fd = open(path, O_RDONLY); if (fd < 0) { if (!must_exist && errno == ENOENT) { + if (!istate->repo) + istate->repo = the_repository; set_new_index_sparsity(istate); return 0; } diff --git a/rebase-interactive.c b/rebase-interactive.c index 87649d0c01..7407c59319 100644 --- a/rebase-interactive.c +++ b/rebase-interactive.c @@ -54,9 +54,12 @@ void append_todo_help(int command_count, "l, label <label> = label current HEAD with a name\n" "t, reset <label> = reset HEAD to a label\n" "m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]\n" -". create a merge commit using the original merge commit's\n" -". message (or the oneline, if no original merge commit was\n" -". specified); use -c <commit> to reword the commit message\n" +" create a merge commit using the original merge commit's\n" +" message (or the oneline, if no original merge commit was\n" +" specified); use -c <commit> to reword the commit message\n" +"u, update-ref <ref> = track a placeholder for the <ref> to be updated\n" +" to this position in the new commits. The <ref> is\n" +" updated at the end of the rebase\n" "\n" "These lines can be re-ordered; they are executed from top to bottom.\n"); unsigned edit_todo = !(shortrevisions && shortonto); @@ -143,6 +146,12 @@ int edit_todo_list(struct repository *r, struct todo_list *todo_list, return -4; } + /* + * See if branches need to be added or removed from the update-refs + * file based on the new todo list. + */ + todo_list_filter_update_refs(r, new_todo); + return 0; } @@ -20,6 +20,7 @@ #include "repository.h" #include "sigchain.h" #include "date.h" +#include "commit.h" /* * List of all available backends @@ -56,6 +57,88 @@ static unsigned char refname_disposition[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 4 }; +struct ref_namespace_info ref_namespace[] = { + [NAMESPACE_HEAD] = { + .ref = "HEAD", + .decoration = DECORATION_REF_HEAD, + .exact = 1, + }, + [NAMESPACE_BRANCHES] = { + .ref = "refs/heads/", + .decoration = DECORATION_REF_LOCAL, + }, + [NAMESPACE_TAGS] = { + .ref = "refs/tags/", + .decoration = DECORATION_REF_TAG, + }, + [NAMESPACE_REMOTE_REFS] = { + /* + * The default refspec for new remotes copies refs from + * refs/heads/ on the remote into refs/remotes/<remote>/. + * As such, "refs/remotes/" has special handling. + */ + .ref = "refs/remotes/", + .decoration = DECORATION_REF_REMOTE, + }, + [NAMESPACE_STASH] = { + /* + * The single ref "refs/stash" stores the latest stash. + * Older stashes can be found in the reflog. + */ + .ref = "refs/stash", + .exact = 1, + .decoration = DECORATION_REF_STASH, + }, + [NAMESPACE_REPLACE] = { + /* + * This namespace allows Git to act as if one object ID + * points to the content of another. Unlike the other + * ref namespaces, this one can be changed by the + * GIT_REPLACE_REF_BASE environment variable. This + * .namespace value will be overwritten in setup_git_env(). + */ + .ref = "refs/replace/", + .decoration = DECORATION_GRAFTED, + }, + [NAMESPACE_NOTES] = { + /* + * The refs/notes/commit ref points to the tip of a + * parallel commit history that adds metadata to commits + * in the normal history. This ref can be overwritten + * by the core.notesRef config variable or the + * GIT_NOTES_REFS environment variable. + */ + .ref = "refs/notes/commit", + .exact = 1, + }, + [NAMESPACE_PREFETCH] = { + /* + * Prefetch refs are written by the background 'fetch' + * maintenance task. It allows faster foreground fetches + * by advertising these previously-downloaded tips without + * updating refs/remotes/ without user intervention. + */ + .ref = "refs/prefetch/", + }, + [NAMESPACE_REWRITTEN] = { + /* + * Rewritten refs are used by the 'label' command in the + * sequencer. These are particularly useful during an + * interactive rebase that uses the 'merge' command. + */ + .ref = "refs/rewritten/", + }, +}; + +void update_ref_namespace(enum ref_namespace namespace, char *ref) +{ + struct ref_namespace_info *info = &ref_namespace[namespace]; + if (info->ref_updated) + free(info->ref); + info->ref = ref; + info->ref_updated = 1; +} + /* * Try to read one refname component from the front of refname. * Return the length of the component found, or -1 if the component is @@ -455,11 +538,16 @@ void normalize_glob_ref(struct string_list_item *item, const char *prefix, if (*pattern == '/') BUG("pattern must not start with '/'"); - if (prefix) { + if (prefix) strbuf_addstr(&normalized_pattern, prefix); - } - else if (!starts_with(pattern, "refs/")) + else if (!starts_with(pattern, "refs/") && + strcmp(pattern, "HEAD")) strbuf_addstr(&normalized_pattern, "refs/"); + /* + * NEEDSWORK: Special case other symrefs such as REBASE_HEAD, + * MERGE_HEAD, etc. + */ + strbuf_addstr(&normalized_pattern, pattern); strbuf_strip_suffix(&normalized_pattern, "/"); @@ -1524,6 +1612,7 @@ int refs_for_each_fullref_in(struct ref_store *refs, const char *prefix, int for_each_replace_ref(struct repository *r, each_repo_ref_fn fn, void *cb_data) { + const char *git_replace_ref_base = ref_namespace[NAMESPACE_REPLACE].ref; return do_for_each_repo_ref(r, git_replace_ref_base, fn, strlen(git_replace_ref_base), DO_FOR_EACH_INCLUDE_BROKEN, cb_data); @@ -2,6 +2,7 @@ #define REFS_H #include "cache.h" +#include "commit.h" struct object_id; struct ref_store; @@ -930,4 +931,49 @@ struct ref_store *get_main_ref_store(struct repository *r); struct ref_store *get_submodule_ref_store(const char *submodule); struct ref_store *get_worktree_ref_store(const struct worktree *wt); +/* + * Some of the names specified by refs have special meaning to Git. + * Organize these namespaces in a comon 'ref_namespace' array for + * reference from multiple places in the codebase. + */ + +struct ref_namespace_info { + char *ref; + enum decoration_type decoration; + + /* + * If 'exact' is true, then we must match the 'ref' exactly. + * Otherwise, use a prefix match. + * + * 'ref_updated' is for internal use. It represents whether the + * 'ref' value was replaced from its original literal version. + */ + unsigned exact:1, + ref_updated:1; +}; + +enum ref_namespace { + NAMESPACE_HEAD, + NAMESPACE_BRANCHES, + NAMESPACE_TAGS, + NAMESPACE_REMOTE_REFS, + NAMESPACE_STASH, + NAMESPACE_REPLACE, + NAMESPACE_NOTES, + NAMESPACE_PREFETCH, + NAMESPACE_REWRITTEN, + + /* Must be last */ + NAMESPACE__COUNT +}; + +/* See refs.c for the contents of this array. */ +extern struct ref_namespace_info ref_namespace[NAMESPACE__COUNT]; + +/* + * Some ref namespaces can be modified by config values or environment + * variables. Modify a namespace as specified by its ref_namespace key. + */ +void update_ref_namespace(enum ref_namespace namespace, char *ref); + #endif /* REFS_H */ @@ -69,7 +69,7 @@ int valid_remote_name(const char *name); struct strvec; /* * Determine what <prefix> values to pass to the peer in ref-prefix lines - * (see Documentation/technical/protocol-v2.txt). + * (see linkgit:gitprotocol-v2[5]). */ void refspec_ref_prefixes(const struct refspec *rs, struct strvec *ref_prefixes); diff --git a/reftable/reader.c b/reftable/reader.c index 54b4025105..b4db23ce18 100644 --- a/reftable/reader.c +++ b/reftable/reader.c @@ -443,7 +443,7 @@ static int reader_start(struct reftable_reader *r, struct table_iter *ti, return reader_table_iter_at(r, ti, off, typ); } -static int reader_seek_linear(struct reftable_reader *r, struct table_iter *ti, +static int reader_seek_linear(struct table_iter *ti, struct reftable_record *want) { struct reftable_record rec = @@ -510,7 +510,7 @@ static int reader_seek_indexed(struct reftable_reader *r, if (err < 0) goto done; - err = reader_seek_linear(r, &index_iter, &want_index); + err = reader_seek_linear(&index_iter, &want_index); while (1) { err = table_iter_next(&index_iter, &index_result); table_iter_block_done(&index_iter); @@ -570,7 +570,7 @@ static int reader_seek_internal(struct reftable_reader *r, err = reader_start(r, &ti, reftable_record_type(rec), 0); if (err < 0) return err; - err = reader_seek_linear(r, &ti, rec); + err = reader_seek_linear(&ti, rec); if (err < 0) return err; else { diff --git a/remote-curl.c b/remote-curl.c index b8758757ec..72dfb8fb86 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1286,6 +1286,29 @@ static void parse_fetch(struct strbuf *buf) strbuf_reset(buf); } +static void parse_get(const char *arg) +{ + struct strbuf url = STRBUF_INIT; + struct strbuf path = STRBUF_INIT; + const char *space; + + space = strchr(arg, ' '); + + if (!space) + die(_("protocol error: expected '<url> <path>', missing space")); + + strbuf_add(&url, arg, space - arg); + strbuf_addstr(&path, space + 1); + + if (http_get_file(url.buf, path.buf, NULL)) + die(_("failed to download file at URL '%s'"), url.buf); + + strbuf_release(&url); + strbuf_release(&path); + printf("\n"); + fflush(stdout); +} + static int push_dav(int nr_spec, const char **specs) { struct child_process child = CHILD_PROCESS_INIT; @@ -1564,9 +1587,14 @@ int cmd_main(int argc, const char **argv) printf("unsupported\n"); fflush(stdout); + } else if (skip_prefix(buf.buf, "get ", &arg)) { + parse_get(arg); + fflush(stdout); + } else if (!strcmp(buf.buf, "capabilities")) { printf("stateless-connect\n"); printf("fetch\n"); + printf("get\n"); printf("option\n"); printf("push\n"); printf("check-connectivity\n"); @@ -11,7 +11,6 @@ #include "dir.h" #include "tag.h" #include "string-list.h" -#include "mergesort.h" #include "strvec.h" #include "commit-reach.h" #include "advice.h" @@ -1082,27 +1081,6 @@ void free_refs(struct ref *ref) } } -int ref_compare_name(const void *va, const void *vb) -{ - const struct ref *a = va, *b = vb; - return strcmp(a->name, b->name); -} - -static void *ref_list_get_next(const void *a) -{ - return ((const struct ref *)a)->next; -} - -static void ref_list_set_next(void *a, void *next) -{ - ((struct ref *)a)->next = next; -} - -void sort_ref_list(struct ref **l, int (*cmp)(const void *, const void *)) -{ - *l = llist_mergesort(*l, ref_list_get_next, ref_list_set_next, cmp); -} - int count_refspec_match(const char *pattern, struct ref *refs, struct ref **matched_ref) @@ -2169,6 +2147,9 @@ static int stat_branch_pair(const char *branch_name, const char *base, struct object_id oid; struct commit *ours, *theirs; struct rev_info revs; + struct setup_revision_opt opt = { + .free_removed_argv_elements = 1, + }; struct strvec argv = STRVEC_INIT; /* Cannot stat if what we used to build on no longer exists */ @@ -2203,7 +2184,7 @@ static int stat_branch_pair(const char *branch_name, const char *base, strvec_push(&argv, "--"); repo_init_revisions(the_repository, &revs, NULL); - setup_revisions(argv.nr, argv.v, &revs, NULL); + setup_revisions(argv.nr, argv.v, &revs, &opt); if (prepare_revision_walk(&revs)) die(_("revision walk setup failed")); @@ -207,9 +207,7 @@ struct ref *find_ref_by_name(const struct ref *list, const char *name); struct ref *alloc_ref(const char *name); struct ref *copy_ref(const struct ref *ref); struct ref *copy_ref_list(const struct ref *ref); -void sort_ref_list(struct ref **, int (*cmp)(const void *, const void *)); int count_refspec_match(const char *, struct ref *refs, struct ref **matched_ref); -int ref_compare_name(const void *, const void *); int check_ref_type(const struct ref *ref, int flags); diff --git a/repo-settings.c b/repo-settings.c index 2dfcb2b654..43bc881bfc 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -11,6 +11,13 @@ static void repo_cfg_bool(struct repository *r, const char *key, int *dest, *dest = def; } +static void repo_cfg_int(struct repository *r, const char *key, int *dest, + int def) +{ + if (repo_config_get_int(r, key, dest)) + *dest = def; +} + void prepare_repo_settings(struct repository *r) { int experimental; @@ -42,11 +49,14 @@ void prepare_repo_settings(struct repository *r) r->settings.core_untracked_cache = UNTRACKED_CACHE_WRITE; } - /* Boolean config or default, does not cascade (simple) */ + /* Commit graph config or default, does not cascade (simple) */ repo_cfg_bool(r, "core.commitgraph", &r->settings.core_commit_graph, 1); + repo_cfg_int(r, "commitgraph.generationversion", &r->settings.commit_graph_generation_version, 2); repo_cfg_bool(r, "commitgraph.readchangedpaths", &r->settings.commit_graph_read_changed_paths, 1); repo_cfg_bool(r, "gc.writecommitgraph", &r->settings.gc_write_commit_graph, 1); repo_cfg_bool(r, "fetch.writecommitgraph", &r->settings.fetch_write_commit_graph, 0); + + /* Boolean config or default, does not cascade (simple) */ repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); diff --git a/repository.h b/repository.h index 6cc661e5a4..797f471cce 100644 --- a/repository.h +++ b/repository.h @@ -30,6 +30,7 @@ struct repo_settings { int initialized; int core_commit_graph; + int commit_graph_generation_version; int commit_graph_read_changed_paths; int gc_write_commit_graph; int fetch_write_commit_graph; diff --git a/revision.c b/revision.c index 0c6e26cd9c..ee702e498a 100644 --- a/revision.c +++ b/revision.c @@ -1105,7 +1105,7 @@ static int process_parents(struct rev_info *revs, struct commit *commit, struct commit_list **list, struct prio_queue *queue) { struct commit_list *parent = commit->parents; - unsigned left_flag; + unsigned pass_flags; if (commit->object.flags & ADDED) return 0; @@ -1160,7 +1160,7 @@ static int process_parents(struct rev_info *revs, struct commit *commit, if (revs->no_walk) return 0; - left_flag = (commit->object.flags & SYMMETRIC_LEFT); + pass_flags = (commit->object.flags & (SYMMETRIC_LEFT | ANCESTRY_PATH)); for (parent = commit->parents; parent; parent = parent->next) { struct commit *p = parent->item; @@ -1181,7 +1181,7 @@ static int process_parents(struct rev_info *revs, struct commit *commit, if (!*slot) *slot = *revision_sources_at(revs->sources, commit); } - p->object.flags |= left_flag; + p->object.flags |= pass_flags; if (!(p->object.flags & SEEN)) { p->object.flags |= (SEEN | NOT_USER_GIVEN); if (list) @@ -1304,13 +1304,24 @@ static int still_interesting(struct commit_list *src, timestamp_t date, int slop } /* - * "rev-list --ancestry-path A..B" computes commits that are ancestors - * of B but not ancestors of A but further limits the result to those - * that are descendants of A. This takes the list of bottom commits and - * the result of "A..B" without --ancestry-path, and limits the latter - * further to the ones that can reach one of the commits in "bottom". + * "rev-list --ancestry-path=C_0 [--ancestry-path=C_1 ...] A..B" + * computes commits that are ancestors of B but not ancestors of A but + * further limits the result to those that have any of C in their + * ancestry path (i.e. are either ancestors of any of C, descendants + * of any of C, or are any of C). If --ancestry-path is specified with + * no commit, we use all bottom commits for C. + * + * Before this function is called, ancestors of C will have already + * been marked with ANCESTRY_PATH previously. + * + * This takes the list of bottom commits and the result of "A..B" + * without --ancestry-path, and limits the latter further to the ones + * that have any of C in their ancestry path. Since the ancestors of C + * have already been marked (a prerequisite of this function), we just + * need to mark the descendants, then exclude any commit that does not + * have any of these marks. */ -static void limit_to_ancestry(struct commit_list *bottom, struct commit_list *list) +static void limit_to_ancestry(struct commit_list *bottoms, struct commit_list *list) { struct commit_list *p; struct commit_list *rlist = NULL; @@ -1323,7 +1334,7 @@ static void limit_to_ancestry(struct commit_list *bottom, struct commit_list *li for (p = list; p; p = p->next) commit_list_insert(p->item, &rlist); - for (p = bottom; p; p = p->next) + for (p = bottoms; p; p = p->next) p->item->object.flags |= TMP_MARK; /* @@ -1356,38 +1367,39 @@ static void limit_to_ancestry(struct commit_list *bottom, struct commit_list *li */ /* - * The ones that are not marked with TMP_MARK are uninteresting + * The ones that are not marked with either TMP_MARK or + * ANCESTRY_PATH are uninteresting */ for (p = list; p; p = p->next) { struct commit *c = p->item; - if (c->object.flags & TMP_MARK) + if (c->object.flags & (TMP_MARK | ANCESTRY_PATH)) continue; c->object.flags |= UNINTERESTING; } - /* We are done with the TMP_MARK */ + /* We are done with TMP_MARK and ANCESTRY_PATH */ for (p = list; p; p = p->next) - p->item->object.flags &= ~TMP_MARK; - for (p = bottom; p; p = p->next) - p->item->object.flags &= ~TMP_MARK; + p->item->object.flags &= ~(TMP_MARK | ANCESTRY_PATH); + for (p = bottoms; p; p = p->next) + p->item->object.flags &= ~(TMP_MARK | ANCESTRY_PATH); free_commit_list(rlist); } /* - * Before walking the history, keep the set of "negative" refs the - * caller has asked to exclude. + * Before walking the history, add the set of "negative" refs the + * caller has asked to exclude to the bottom list. * * This is used to compute "rev-list --ancestry-path A..B", as we need * to filter the result of "A..B" further to the ones that can actually * reach A. */ -static struct commit_list *collect_bottom_commits(struct commit_list *list) +static void collect_bottom_commits(struct commit_list *list, + struct commit_list **bottom) { - struct commit_list *elem, *bottom = NULL; + struct commit_list *elem; for (elem = list; elem; elem = elem->next) if (elem->item->object.flags & BOTTOM) - commit_list_insert(elem->item, &bottom); - return bottom; + commit_list_insert(elem->item, bottom); } /* Assumes either left_only or right_only is set */ @@ -1414,12 +1426,12 @@ static int limit_list(struct rev_info *revs) struct commit_list *original_list = revs->commits; struct commit_list *newlist = NULL; struct commit_list **p = &newlist; - struct commit_list *bottom = NULL; struct commit *interesting_cache = NULL; - if (revs->ancestry_path) { - bottom = collect_bottom_commits(original_list); - if (!bottom) + if (revs->ancestry_path_implicit_bottoms) { + collect_bottom_commits(original_list, + &revs->ancestry_path_bottoms); + if (!revs->ancestry_path_bottoms) die("--ancestry-path given but there are no bottom commits"); } @@ -1464,9 +1476,8 @@ static int limit_list(struct rev_info *revs) if (revs->left_only || revs->right_only) limit_left_right(newlist, revs); - if (bottom) - limit_to_ancestry(bottom, newlist); - free_commit_list(bottom); + if (revs->ancestry_path) + limit_to_ancestry(revs->ancestry_path_bottoms, newlist); /* * Check if any commits have become TREESAME by some of their parents @@ -2213,7 +2224,7 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg const struct setup_revision_opt* opt) { const char *arg = argv[0]; - const char *optarg; + const char *optarg = NULL; int argcount; const unsigned hexsz = the_hash_algo->hexsz; @@ -2284,6 +2295,23 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg revs->ancestry_path = 1; revs->simplify_history = 0; revs->limited = 1; + revs->ancestry_path_implicit_bottoms = 1; + } else if (skip_prefix(arg, "--ancestry-path=", &optarg)) { + struct commit *c; + struct object_id oid; + const char *msg = _("could not get commit for ancestry-path argument %s"); + + revs->ancestry_path = 1; + revs->simplify_history = 0; + revs->limited = 1; + + if (repo_get_oid_committish(revs->repo, optarg, &oid)) + return error(msg, optarg); + get_reference(revs, optarg, &oid, ANCESTRY_PATH); + c = lookup_commit_reference(revs->repo, &oid); + if (!c) + return error(msg, optarg); + commit_list_insert(c, &revs->ancestry_path_bottoms); } else if (!strcmp(arg, "-g") || !strcmp(arg, "--walk-reflogs")) { init_reflog_walk(&revs->reflog_info); } else if (!strcmp(arg, "--default")) { @@ -2784,6 +2812,8 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s const char *arg = argv[i]; if (strcmp(arg, "--")) continue; + if (opt && opt->free_removed_argv_elements) + free((char *)argv[i]); argv[i] = NULL; argc = i; if (argv[i + 1]) @@ -2991,6 +3021,7 @@ static void release_revisions_topo_walk_info(struct topo_walk_info *info); void release_revisions(struct rev_info *revs) { free_commit_list(revs->commits); + free_commit_list(revs->ancestry_path_bottoms); object_array_clear(&revs->pending); object_array_clear(&revs->boundary_commits); release_revisions_cmdline(&revs->cmdline); @@ -3791,51 +3822,6 @@ int rewrite_parents(struct rev_info *revs, struct commit *commit, return 0; } -static int commit_rewrite_person(struct strbuf *buf, const char *what, struct string_list *mailmap) -{ - char *person, *endp; - size_t len, namelen, maillen; - const char *name; - const char *mail; - struct ident_split ident; - - person = strstr(buf->buf, what); - if (!person) - return 0; - - person += strlen(what); - endp = strchr(person, '\n'); - if (!endp) - return 0; - - len = endp - person; - - if (split_ident_line(&ident, person, len)) - return 0; - - mail = ident.mail_begin; - maillen = ident.mail_end - ident.mail_begin; - name = ident.name_begin; - namelen = ident.name_end - ident.name_begin; - - if (map_user(mailmap, &mail, &maillen, &name, &namelen)) { - struct strbuf namemail = STRBUF_INIT; - - strbuf_addf(&namemail, "%.*s <%.*s>", - (int)namelen, name, (int)maillen, mail); - - strbuf_splice(buf, ident.name_begin - buf->buf, - ident.mail_end - ident.name_begin + 1, - namemail.buf, namemail.len); - - strbuf_release(&namemail); - - return 1; - } - - return 0; -} - static int commit_match(struct commit *commit, struct rev_info *opt) { int retval; @@ -3868,11 +3854,12 @@ static int commit_match(struct commit *commit, struct rev_info *opt) strbuf_addstr(&buf, message); if (opt->grep_filter.header_list && opt->mailmap) { + const char *commit_headers[] = { "author ", "committer ", NULL }; + if (!buf.len) strbuf_addstr(&buf, message); - commit_rewrite_person(&buf, "\nauthor ", opt->mailmap); - commit_rewrite_person(&buf, "\ncommitter ", opt->mailmap); + apply_mailmap_to_header(&buf, commit_headers, opt->mailmap); } /* Append "fake" message parts as needed */ diff --git a/revision.h b/revision.h index e576845cdd..61a9b1316b 100644 --- a/revision.h +++ b/revision.h @@ -48,6 +48,7 @@ */ #define NOT_USER_GIVEN (1u<<25) #define TRACK_LINEAR (1u<<26) +#define ANCESTRY_PATH (1u<<27) #define ALL_REV_FLAGS (((1u<<11)-1) | NOT_USER_GIVEN | TRACK_LINEAR | PULL_MERGE) #define DECORATE_SHORT_REFS 1 @@ -164,6 +165,13 @@ struct rev_info { cherry_mark:1, bisect:1, ancestry_path:1, + + /* True if --ancestry-path was specified without an + * argument. The bottom revisions are implicitly + * the arguments in this case. + */ + ancestry_path_implicit_bottoms:1, + first_parent_only:1, exclude_first_parent_only:1, line_level_traverse:1, @@ -306,6 +314,7 @@ struct rev_info { struct saved_parents *saved_parents_slab; struct commit_list *previous_parents; + struct commit_list *ancestry_path_bottoms; const char *break_bar; struct revision_sources *sources; @@ -375,7 +384,8 @@ struct setup_revision_opt { const char *def; void (*tweak)(struct rev_info *, struct setup_revision_opt *); unsigned int assume_dashdash:1, - allow_exclude_promisor_objects:1; + allow_exclude_promisor_objects:1, + free_removed_argv_elements:1; unsigned revarg_opt; }; int setup_revisions(int argc, const char **argv, struct rev_info *revs, diff --git a/run-command.c b/run-command.c index 14f17830f5..5ec3a46dcc 100644 --- a/run-command.c +++ b/run-command.c @@ -10,6 +10,7 @@ #include "config.h" #include "packfile.h" #include "hook.h" +#include "compat/nonblock.h" void child_process_init(struct child_process *child) { @@ -1364,12 +1365,25 @@ static int pump_io_round(struct io_pump *slots, int nr, struct pollfd *pfd) continue; if (io->type == POLLOUT) { - ssize_t len = xwrite(io->fd, - io->u.out.buf, io->u.out.len); + ssize_t len; + + /* + * Don't use xwrite() here. It loops forever on EAGAIN, + * and we're in our own poll() loop here. + * + * Note that we lose xwrite()'s handling of MAX_IO_SIZE + * and EINTR, so we have to implement those ourselves. + */ + len = write(io->fd, io->u.out.buf, + io->u.out.len <= MAX_IO_SIZE ? + io->u.out.len : MAX_IO_SIZE); if (len < 0) { - io->error = errno; - close(io->fd); - io->fd = -1; + if (errno != EINTR && errno != EAGAIN && + errno != ENOSPC) { + io->error = errno; + close(io->fd); + io->fd = -1; + } } else { io->u.out.buf += len; io->u.out.len -= len; @@ -1438,6 +1452,15 @@ int pipe_command(struct child_process *cmd, return -1; if (in) { + if (enable_pipe_nonblock(cmd->in) < 0) { + error_errno("unable to make pipe non-blocking"); + close(cmd->in); + if (out) + close(cmd->out); + if (err) + close(cmd->err); + return -1; + } io[nr].fd = cmd->in; io[nr].type = POLLOUT; io[nr].u.out.buf = in; diff --git a/sequencer.c b/sequencer.c index 61a8e0020d..79dad522f5 100644 --- a/sequencer.c +++ b/sequencer.c @@ -35,6 +35,8 @@ #include "commit-reach.h" #include "rebase-interactive.h" #include "reset.h" +#include "branch.h" +#include "log-tree.h" #define GIT_REFLOG_ACTION "GIT_REFLOG_ACTION" @@ -148,6 +150,20 @@ static GIT_PATH_FUNC(rebase_path_squash_onto, "rebase-merge/squash-onto") static GIT_PATH_FUNC(rebase_path_refs_to_delete, "rebase-merge/refs-to-delete") /* + * The update-refs file stores a list of refs that will be updated at the end + * of the rebase sequence. The 'update-ref <ref>' commands in the todo file + * update the OIDs for the refs in this file, but the refs are not updated + * until the end of the rebase sequence. + * + * rebase_path_update_refs() returns the path to this file for a given + * worktree directory. For the current worktree, pass the_repository->gitdir. + */ +static char *rebase_path_update_refs(const char *wt_git_dir) +{ + return xstrfmt("%s/rebase-merge/update-refs", wt_git_dir); +} + +/* * The following files are written by git-rebase just after parsing the * command-line. */ @@ -169,6 +185,30 @@ static GIT_PATH_FUNC(rebase_path_no_reschedule_failed_exec, "rebase-merge/no-res static GIT_PATH_FUNC(rebase_path_drop_redundant_commits, "rebase-merge/drop_redundant_commits") static GIT_PATH_FUNC(rebase_path_keep_redundant_commits, "rebase-merge/keep_redundant_commits") +/** + * A 'struct update_refs_record' represents a value in the update-refs + * list. We use a string_list to map refs to these (before, after) pairs. + */ +struct update_ref_record { + struct object_id before; + struct object_id after; +}; + +static struct update_ref_record *init_update_ref_record(const char *ref) +{ + struct update_ref_record *rec; + + CALLOC_ARRAY(rec, 1); + + oidcpy(&rec->before, null_oid()); + oidcpy(&rec->after, null_oid()); + + /* This may fail, but that's fine, we will keep the null OID. */ + read_ref(ref, &rec->before); + + return rec; +} + static int git_sequencer_config(const char *k, const char *v, void *cb) { struct replay_opts *opts = cb; @@ -497,7 +537,7 @@ static struct tree *empty_tree(struct repository *r) static int error_dirty_index(struct repository *repo, struct replay_opts *opts) { if (repo_read_index_unmerged(repo)) - return error_resolve_conflict(_(action_name(opts))); + return error_resolve_conflict(action_name(opts)); error(_("your local changes would be overwritten by %s."), _(action_name(opts))); @@ -535,7 +575,7 @@ static int fast_forward_to(struct repository *r, if (checkout_fast_forward(r, from, to, 1)) return -1; /* the callee should have complained already */ - strbuf_addf(&sb, _("%s: fast-forward"), _(action_name(opts))); + strbuf_addf(&sb, "%s: fast-forward", action_name(opts)); transaction = ref_transaction_begin(&err); if (!transaction || @@ -1689,20 +1729,21 @@ static struct { char c; const char *str; } todo_command_info[] = { - { 'p', "pick" }, - { 0, "revert" }, - { 'e', "edit" }, - { 'r', "reword" }, - { 'f', "fixup" }, - { 's', "squash" }, - { 'x', "exec" }, - { 'b', "break" }, - { 'l', "label" }, - { 't', "reset" }, - { 'm', "merge" }, - { 0, "noop" }, - { 'd', "drop" }, - { 0, NULL } + [TODO_PICK] = { 'p', "pick" }, + [TODO_REVERT] = { 0, "revert" }, + [TODO_EDIT] = { 'e', "edit" }, + [TODO_REWORD] = { 'r', "reword" }, + [TODO_FIXUP] = { 'f', "fixup" }, + [TODO_SQUASH] = { 's', "squash" }, + [TODO_EXEC] = { 'x', "exec" }, + [TODO_BREAK] = { 'b', "break" }, + [TODO_LABEL] = { 'l', "label" }, + [TODO_RESET] = { 't', "reset" }, + [TODO_MERGE] = { 'm', "merge" }, + [TODO_UPDATE_REF] = { 'u', "update-ref" }, + [TODO_NOOP] = { 0, "noop" }, + [TODO_DROP] = { 'd', "drop" }, + [TODO_COMMENT] = { 0, NULL }, }; static const char *command_to_string(const enum todo_command command) @@ -2381,7 +2422,7 @@ static int read_and_refresh_cache(struct repository *r, if (repo_read_index(r) < 0) { rollback_lock_file(&index_lock); return error(_("git %s: failed to read the index"), - _(action_name(opts))); + action_name(opts)); } refresh_index(r->index, REFRESH_QUIET|REFRESH_UNMERGED, NULL, NULL, NULL); @@ -2389,7 +2430,7 @@ static int read_and_refresh_cache(struct repository *r, if (write_locked_index(r->index, &index_lock, COMMIT_LOCK | SKIP_IF_UNCHANGED)) { return error(_("git %s: failed to refresh the index"), - _(action_name(opts))); + action_name(opts)); } } @@ -2481,7 +2522,7 @@ static int parse_insn_line(struct repository *r, struct todo_item *item, command_to_string(item->command)); if (item->command == TODO_EXEC || item->command == TODO_LABEL || - item->command == TODO_RESET) { + item->command == TODO_RESET || item->command == TODO_UPDATE_REF) { item->commit = NULL; item->arg_offset = bol - buf; item->arg_len = (int)(eol - bol); @@ -3712,7 +3753,7 @@ static int do_reset(struct repository *r, init_checkout_metadata(&unpack_tree_opts.meta, name, &oid, NULL); if (repo_read_index_unmerged(r)) { - ret = error_resolve_conflict(_(action_name(opts))); + ret = error_resolve_conflict(action_name(opts)); goto cleanup; } @@ -4081,6 +4122,221 @@ leave_merge: return ret; } +static int write_update_refs_state(struct string_list *refs_to_oids) +{ + int result = 0; + struct lock_file lock = LOCK_INIT; + FILE *fp = NULL; + struct string_list_item *item; + char *path; + + if (!refs_to_oids->nr) + return 0; + + path = rebase_path_update_refs(the_repository->gitdir); + + if (safe_create_leading_directories(path)) { + result = error(_("unable to create leading directories of %s"), + path); + goto cleanup; + } + + if (hold_lock_file_for_update(&lock, path, 0) < 0) { + result = error(_("another 'rebase' process appears to be running; " + "'%s.lock' already exists"), + path); + goto cleanup; + } + + fp = fdopen_lock_file(&lock, "w"); + if (!fp) { + result = error_errno(_("could not open '%s' for writing"), path); + rollback_lock_file(&lock); + goto cleanup; + } + + for_each_string_list_item(item, refs_to_oids) { + struct update_ref_record *rec = item->util; + fprintf(fp, "%s\n%s\n%s\n", item->string, + oid_to_hex(&rec->before), oid_to_hex(&rec->after)); + } + + result = commit_lock_file(&lock); + +cleanup: + free(path); + return result; +} + +/* + * Parse the update-refs file for the current rebase, then remove the + * refs that do not appear in the todo_list (and have not had updated + * values stored) and add refs that are in the todo_list but not + * represented in the update-refs file. + * + * If there are changes to the update-refs list, then write the new state + * to disk. + */ +void todo_list_filter_update_refs(struct repository *r, + struct todo_list *todo_list) +{ + int i; + int updated = 0; + struct string_list update_refs = STRING_LIST_INIT_DUP; + + sequencer_get_update_refs_state(r->gitdir, &update_refs); + + /* + * For each item in the update_refs list, if it has no updated + * value and does not appear in the todo_list, then remove it + * from the update_refs list. + */ + for (i = 0; i < update_refs.nr; i++) { + int j; + int found = 0; + const char *ref = update_refs.items[i].string; + size_t reflen = strlen(ref); + struct update_ref_record *rec = update_refs.items[i].util; + + /* OID already stored as updated. */ + if (!is_null_oid(&rec->after)) + continue; + + for (j = 0; !found && j < todo_list->total_nr; j++) { + struct todo_item *item = &todo_list->items[j]; + const char *arg = todo_list->buf.buf + item->arg_offset; + + if (item->command != TODO_UPDATE_REF) + continue; + + if (item->arg_len != reflen || + strncmp(arg, ref, reflen)) + continue; + + found = 1; + } + + if (!found) { + free(update_refs.items[i].string); + free(update_refs.items[i].util); + + update_refs.nr--; + MOVE_ARRAY(update_refs.items + i, update_refs.items + i + 1, update_refs.nr - i); + + updated = 1; + i--; + } + } + + /* + * For each todo_item, check if its ref is in the update_refs list. + * If not, then add it as an un-updated ref. + */ + for (i = 0; i < todo_list->total_nr; i++) { + struct todo_item *item = &todo_list->items[i]; + const char *arg = todo_list->buf.buf + item->arg_offset; + int j, found = 0; + + if (item->command != TODO_UPDATE_REF) + continue; + + for (j = 0; !found && j < update_refs.nr; j++) { + const char *ref = update_refs.items[j].string; + + found = strlen(ref) == item->arg_len && + !strncmp(ref, arg, item->arg_len); + } + + if (!found) { + struct string_list_item *inserted; + struct strbuf argref = STRBUF_INIT; + + strbuf_add(&argref, arg, item->arg_len); + inserted = string_list_insert(&update_refs, argref.buf); + inserted->util = init_update_ref_record(argref.buf); + strbuf_release(&argref); + updated = 1; + } + } + + if (updated) + write_update_refs_state(&update_refs); + string_list_clear(&update_refs, 1); +} + +static int do_update_ref(struct repository *r, const char *refname) +{ + struct string_list_item *item; + struct string_list list = STRING_LIST_INIT_DUP; + + if (sequencer_get_update_refs_state(r->gitdir, &list)) + return -1; + + for_each_string_list_item(item, &list) { + if (!strcmp(item->string, refname)) { + struct update_ref_record *rec = item->util; + if (read_ref("HEAD", &rec->after)) + return -1; + break; + } + } + + write_update_refs_state(&list); + string_list_clear(&list, 1); + return 0; +} + +static int do_update_refs(struct repository *r, int quiet) +{ + int res = 0; + struct string_list_item *item; + struct string_list refs_to_oids = STRING_LIST_INIT_DUP; + struct ref_store *refs = get_main_ref_store(r); + struct strbuf update_msg = STRBUF_INIT; + struct strbuf error_msg = STRBUF_INIT; + + if ((res = sequencer_get_update_refs_state(r->gitdir, &refs_to_oids))) + return res; + + for_each_string_list_item(item, &refs_to_oids) { + struct update_ref_record *rec = item->util; + int loop_res; + + loop_res = refs_update_ref(refs, "rewritten during rebase", + item->string, + &rec->after, &rec->before, + 0, UPDATE_REFS_MSG_ON_ERR); + res |= loop_res; + + if (quiet) + continue; + + if (loop_res) + strbuf_addf(&error_msg, "\t%s\n", item->string); + else + strbuf_addf(&update_msg, "\t%s\n", item->string); + } + + if (!quiet && + (update_msg.len || error_msg.len)) { + fprintf(stderr, + _("Updated the following refs with %s:\n%s"), + "--update-refs", + update_msg.buf); + + if (res) + fprintf(stderr, + _("Failed to update the following refs with %s:\n%s"), + "--update-refs", + error_msg.buf); + } + + string_list_clear(&refs_to_oids, 1); + strbuf_release(&update_msg); + strbuf_release(&error_msg); + return res; +} + static int is_final_fixup(struct todo_list *todo_list) { int i = todo_list->current; @@ -4456,6 +4712,12 @@ static int pick_commits(struct repository *r, return error_with_patch(r, item->commit, arg, item->arg_len, opts, res, 0); + } else if (item->command == TODO_UPDATE_REF) { + struct strbuf ref = STRBUF_INIT; + strbuf_add(&ref, arg, item->arg_len); + if ((res = do_update_ref(r, ref.buf))) + reschedule = 1; + strbuf_release(&ref); } else if (!is_noop(item->command)) return error(_("unknown command %d"), item->command); @@ -4591,6 +4853,9 @@ cleanup_head_ref: strbuf_release(&buf); strbuf_release(&head_ref); + + if (do_update_refs(r, opts->quiet)) + return -1; } /* @@ -5638,10 +5903,135 @@ static int skip_unnecessary_picks(struct repository *r, return 0; } +struct todo_add_branch_context { + struct todo_item *items; + size_t items_nr; + size_t items_alloc; + struct strbuf *buf; + struct commit *commit; + struct string_list refs_to_oids; +}; + +static int add_decorations_to_list(const struct commit *commit, + struct todo_add_branch_context *ctx) +{ + const struct name_decoration *decoration = get_name_decoration(&commit->object); + const char *head_ref = resolve_ref_unsafe("HEAD", + RESOLVE_REF_READING, + NULL, + NULL); + + while (decoration) { + struct todo_item *item; + const char *path; + size_t base_offset = ctx->buf->len; + + /* + * If the branch is the current HEAD, then it will be + * updated by the default rebase behavior. + */ + if (head_ref && !strcmp(head_ref, decoration->name)) { + decoration = decoration->next; + continue; + } + + ALLOC_GROW(ctx->items, + ctx->items_nr + 1, + ctx->items_alloc); + item = &ctx->items[ctx->items_nr]; + memset(item, 0, sizeof(*item)); + + /* If the branch is checked out, then leave a comment instead. */ + if ((path = branch_checked_out(decoration->name))) { + item->command = TODO_COMMENT; + strbuf_addf(ctx->buf, "# Ref %s checked out at '%s'\n", + decoration->name, path); + } else { + struct string_list_item *sti; + item->command = TODO_UPDATE_REF; + strbuf_addf(ctx->buf, "%s\n", decoration->name); + + sti = string_list_insert(&ctx->refs_to_oids, + decoration->name); + sti->util = init_update_ref_record(decoration->name); + } + + item->offset_in_buf = base_offset; + item->arg_offset = base_offset; + item->arg_len = ctx->buf->len - base_offset; + ctx->items_nr++; + + decoration = decoration->next; + } + + return 0; +} + +/* + * For each 'pick' command, find out if the commit has a decoration in + * refs/heads/. If so, then add a 'label for-update-refs/' command. + */ +static int todo_list_add_update_ref_commands(struct todo_list *todo_list) +{ + int i, res; + static struct string_list decorate_refs_exclude = STRING_LIST_INIT_NODUP; + static struct string_list decorate_refs_exclude_config = STRING_LIST_INIT_NODUP; + static struct string_list decorate_refs_include = STRING_LIST_INIT_NODUP; + struct decoration_filter decoration_filter = { + .include_ref_pattern = &decorate_refs_include, + .exclude_ref_pattern = &decorate_refs_exclude, + .exclude_ref_config_pattern = &decorate_refs_exclude_config, + }; + struct todo_add_branch_context ctx = { + .buf = &todo_list->buf, + .refs_to_oids = STRING_LIST_INIT_DUP, + }; + + ctx.items_alloc = 2 * todo_list->nr + 1; + ALLOC_ARRAY(ctx.items, ctx.items_alloc); + + string_list_append(&decorate_refs_include, "refs/heads/"); + load_ref_decorations(&decoration_filter, 0); + + for (i = 0; i < todo_list->nr; ) { + struct todo_item *item = &todo_list->items[i]; + + /* insert ith item into new list */ + ALLOC_GROW(ctx.items, + ctx.items_nr + 1, + ctx.items_alloc); + + ctx.items[ctx.items_nr++] = todo_list->items[i++]; + + if (item->commit) { + ctx.commit = item->commit; + add_decorations_to_list(item->commit, &ctx); + } + } + + res = write_update_refs_state(&ctx.refs_to_oids); + + string_list_clear(&ctx.refs_to_oids, 1); + + if (res) { + /* we failed, so clean up the new list. */ + free(ctx.items); + return res; + } + + free(todo_list->items); + todo_list->items = ctx.items; + todo_list->nr = ctx.items_nr; + todo_list->alloc = ctx.items_alloc; + + return 0; +} + int complete_action(struct repository *r, struct replay_opts *opts, unsigned flags, const char *shortrevisions, const char *onto_name, struct commit *onto, const struct object_id *orig_head, struct string_list *commands, unsigned autosquash, + unsigned update_refs, struct todo_list *todo_list) { char shortonto[GIT_MAX_HEXSZ + 1]; @@ -5660,6 +6050,9 @@ int complete_action(struct repository *r, struct replay_opts *opts, unsigned fla item->arg_len = item->arg_offset = item->flags = item->offset_in_buf = 0; } + if (update_refs && todo_list_add_update_ref_commands(todo_list)) + return -1; + if (autosquash && todo_list_rearrange_squash(todo_list)) return -1; @@ -5936,3 +6329,54 @@ int sequencer_determine_whence(struct repository *r, enum commit_whence *whence) return 0; } + +int sequencer_get_update_refs_state(const char *wt_dir, + struct string_list *refs) +{ + int result = 0; + FILE *fp = NULL; + struct strbuf ref = STRBUF_INIT; + struct strbuf hash = STRBUF_INIT; + struct update_ref_record *rec = NULL; + + char *path = rebase_path_update_refs(wt_dir); + + fp = fopen(path, "r"); + if (!fp) + goto cleanup; + + while (strbuf_getline(&ref, fp) != EOF) { + struct string_list_item *item; + + CALLOC_ARRAY(rec, 1); + + if (strbuf_getline(&hash, fp) == EOF || + get_oid_hex(hash.buf, &rec->before)) { + warning(_("update-refs file at '%s' is invalid"), + path); + result = -1; + goto cleanup; + } + + if (strbuf_getline(&hash, fp) == EOF || + get_oid_hex(hash.buf, &rec->after)) { + warning(_("update-refs file at '%s' is invalid"), + path); + result = -1; + goto cleanup; + } + + item = string_list_insert(refs, ref.buf); + item->util = rec; + rec = NULL; + } + +cleanup: + if (fp) + fclose(fp); + free(path); + free(rec); + strbuf_release(&ref); + strbuf_release(&hash); + return result; +} diff --git a/sequencer.h b/sequencer.h index 698599fe4e..563fe59933 100644 --- a/sequencer.h +++ b/sequencer.h @@ -96,6 +96,7 @@ enum todo_command { TODO_LABEL, TODO_RESET, TODO_MERGE, + TODO_UPDATE_REF, /* commands that do nothing but are counted for reporting progress */ TODO_NOOP, TODO_DROP, @@ -132,6 +133,18 @@ void todo_list_release(struct todo_list *todo_list); const char *todo_item_get_arg(struct todo_list *todo_list, struct todo_item *item); +/* + * Parse the update-refs file for the current rebase, then remove the + * refs that do not appear in the todo_list (and have not had updated + * values stored) and add refs that are in the todo_list but not + * represented in the update-refs file. + * + * If there are changes to the update-refs list, then write the new state + * to disk. + */ +void todo_list_filter_update_refs(struct repository *r, + struct todo_list *todo_list); + /* Call this to setup defaults before parsing command line options */ void sequencer_init_config(struct replay_opts *opts); int sequencer_pick_revisions(struct repository *repo, @@ -167,6 +180,7 @@ int complete_action(struct repository *r, struct replay_opts *opts, unsigned fla const char *shortrevisions, const char *onto_name, struct commit *onto, const struct object_id *orig_head, struct string_list *commands, unsigned autosquash, + unsigned update_refs, struct todo_list *todo_list); int todo_list_rearrange_squash(struct todo_list *todo_list); @@ -233,4 +247,13 @@ void sequencer_post_commit_cleanup(struct repository *r, int verbose); int sequencer_get_last_command(struct repository* r, enum replay_action *action); int sequencer_determine_whence(struct repository *r, enum commit_whence *whence); + +/** + * Append the set of ref-OID pairs that are currently stored for the 'git + * rebase --update-refs' feature if such a rebase is currently happening. + * + * Localized to a worktree's git dir. + */ +int sequencer_get_update_refs_state(const char *wt_dir, struct string_list *refs); + #endif /* SEQUENCER_H */ @@ -1142,16 +1142,17 @@ static int safe_directory_cb(const char *key, const char *value, void *d) * added, for bare ones their git directory. */ static int ensure_valid_ownership(const char *gitfile, - const char *worktree, const char *gitdir) + const char *worktree, const char *gitdir, + struct strbuf *report) { struct safe_directory_data data = { .path = worktree ? worktree : gitdir }; if (!git_env_bool("GIT_TEST_ASSUME_DIFFERENT_OWNER", 0) && - (!gitfile || is_path_owned_by_current_user(gitfile)) && - (!worktree || is_path_owned_by_current_user(worktree)) && - (!gitdir || is_path_owned_by_current_user(gitdir))) + (!gitfile || is_path_owned_by_current_user(gitfile, report)) && + (!worktree || is_path_owned_by_current_user(worktree, report)) && + (!gitdir || is_path_owned_by_current_user(gitdir, report))) return 1; /* @@ -1232,6 +1233,7 @@ enum discovery_result { */ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, struct strbuf *gitdir, + struct strbuf *report, int die_on_error) { const char *env_ceiling_dirs = getenv(CEILING_DIRECTORIES_ENVIRONMENT); @@ -1316,10 +1318,11 @@ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, strbuf_setlen(dir, offset); if (gitdirenv) { enum discovery_result ret; + const char *gitdir_candidate = + gitdir_path ? gitdir_path : gitdirenv; - if (ensure_valid_ownership(gitfile, - dir->buf, - (gitdir_path ? gitdir_path : gitdirenv))) { + if (ensure_valid_ownership(gitfile, dir->buf, + gitdir_candidate, report)) { strbuf_addstr(gitdir, gitdirenv); ret = GIT_DIR_DISCOVERED; } else @@ -1344,7 +1347,7 @@ static enum discovery_result setup_git_directory_gently_1(struct strbuf *dir, if (is_git_directory(dir->buf)) { if (get_allowed_bare_repo() == ALLOWED_BARE_REPO_EXPLICIT) return GIT_DIR_DISALLOWED_BARE; - if (!ensure_valid_ownership(NULL, NULL, dir->buf)) + if (!ensure_valid_ownership(NULL, NULL, dir->buf, report)) return GIT_DIR_INVALID_OWNERSHIP; strbuf_addstr(gitdir, "."); return GIT_DIR_BARE; @@ -1377,7 +1380,7 @@ int discover_git_directory(struct strbuf *commondir, return -1; cwd_len = dir.len; - if (setup_git_directory_gently_1(&dir, gitdir, 0) <= 0) { + if (setup_git_directory_gently_1(&dir, gitdir, NULL, 0) <= 0) { strbuf_release(&dir); return -1; } @@ -1424,7 +1427,7 @@ int discover_git_directory(struct strbuf *commondir, const char *setup_git_directory_gently(int *nongit_ok) { static struct strbuf cwd = STRBUF_INIT; - struct strbuf dir = STRBUF_INIT, gitdir = STRBUF_INIT; + struct strbuf dir = STRBUF_INIT, gitdir = STRBUF_INIT, report = STRBUF_INIT; const char *prefix = NULL; struct repository_format repo_fmt = REPOSITORY_FORMAT_INIT; @@ -1449,7 +1452,7 @@ const char *setup_git_directory_gently(int *nongit_ok) die_errno(_("Unable to read current working directory")); strbuf_addbuf(&dir, &cwd); - switch (setup_git_directory_gently_1(&dir, &gitdir, 1)) { + switch (setup_git_directory_gently_1(&dir, &gitdir, &report, 1)) { case GIT_DIR_EXPLICIT: prefix = setup_explicit_git_dir(gitdir.buf, &cwd, &repo_fmt, nongit_ok); break; @@ -1481,12 +1484,14 @@ const char *setup_git_directory_gently(int *nongit_ok) if (!nongit_ok) { struct strbuf quoted = STRBUF_INIT; + strbuf_complete(&report, '\n'); sq_quote_buf_pretty("ed, dir.buf); die(_("detected dubious ownership in repository at '%s'\n" + "%s" "To add an exception for this directory, call:\n" "\n" "\tgit config --global --add safe.directory %s"), - dir.buf, quoted.buf); + dir.buf, report.buf, quoted.buf); } *nongit_ok = 1; break; @@ -1573,6 +1578,7 @@ const char *setup_git_directory_gently(int *nongit_ok) strbuf_release(&dir); strbuf_release(&gitdir); + strbuf_release(&report); clear_repository_format(&repo_fmt); return prefix; diff --git a/t/Makefile b/t/Makefile index 7f56e52f76..1c80c0c79a 100644 --- a/t/Makefile +++ b/t/Makefile @@ -62,7 +62,7 @@ pre-clean: $(RM) -r '$(TEST_RESULTS_DIRECTORY_SQ)' clean-except-prove-cache: clean-chainlint - $(RM) -r 'trash directory'.* '$(TEST_RESULTS_DIRECTORY_SQ)' + $(RM) -r 'trash directory'.* $(RM) -r valgrind/bin clean: clean-except-prove-cache @@ -366,12 +366,47 @@ excluded as so much relies on it, but this might change in the future. GIT_TEST_SPLIT_INDEX=<boolean> forces split-index mode on the whole test suite. Accept any boolean values that are accepted by git-config. -GIT_TEST_PASSING_SANITIZE_LEAK=<boolean> when compiled with -SANITIZE=leak will run only those tests that have whitelisted -themselves as passing with no memory leaks. Tests can be whitelisted -by setting "TEST_PASSES_SANITIZE_LEAK=true" before sourcing -"test-lib.sh" itself at the top of the test script. This test mode is -used by the "linux-leaks" CI target. +GIT_TEST_PASSING_SANITIZE_LEAK=true skips those tests that haven't +declared themselves as leak-free by setting +"TEST_PASSES_SANITIZE_LEAK=true" before sourcing "test-lib.sh". This +test mode is used by the "linux-leaks" CI target. + +GIT_TEST_PASSING_SANITIZE_LEAK=check checks that our +"TEST_PASSES_SANITIZE_LEAK=true" markings are current. Rather than +skipping those tests that haven't set "TEST_PASSES_SANITIZE_LEAK=true" +before sourcing "test-lib.sh" this mode runs them with +"--invert-exit-code". This is used to check that there's a one-to-one +mapping between "TEST_PASSES_SANITIZE_LEAK=true" and those tests that +pass under "SANITIZE=leak". This is especially useful when testing a +series that fixes various memory leaks with "git rebase -x". + +GIT_TEST_SANITIZE_LEAK_LOG=true will log memory leaks to +"test-results/$TEST_NAME.leak/trace.*" files. The logs include a +"dedup_token" (see +"ASAN_OPTIONS=help=1 ./git") and other options to +make logs +machine-readable. + +With GIT_TEST_SANITIZE_LEAK_LOG=true we'll look at the leak logs +before exiting and exit on failure if the logs showed that we had a +memory leak, even if the test itself would have otherwise passed. This +allows us to catch e.g. missing &&-chaining. This is especially useful +when combined with "GIT_TEST_PASSING_SANITIZE_LEAK", see below. + +GIT_TEST_PASSING_SANITIZE_LEAK=check when combined with "--immediate" +will run to completion faster, and result in the same failing +tests. The only practical reason to run +GIT_TEST_PASSING_SANITIZE_LEAK=check without "--immediate" is to +combine it with "GIT_TEST_SANITIZE_LEAK_LOG=true". If we stop at the +first failing test case our leak logs won't show subsequent leaks we +might have run into. + +GIT_TEST_PASSING_SANITIZE_LEAK=(true|check) will not catch all memory +leaks unless combined with GIT_TEST_SANITIZE_LEAK_LOG=true. Some tests +run "git" (or "test-tool" etc.) without properly checking the exit +code, or git will invoke itself and fail to ferry the abort() exit +code to the original caller. When the two modes are combined we'll +look at the "test-results/$TEST_NAME.leak/trace.*" files at the end of +the test run to see if had memory leaks which the test itself didn't +catch. GIT_TEST_PROTOCOL_VERSION=<n>, when set, makes 'protocol.version' default to n. @@ -935,32 +970,6 @@ see test-lib-functions.sh for the full list and their options. test_done fi - - test_external [<prereq>] <message> <external> <script> - - Execute a <script> with an <external> interpreter (like perl). This - was added for tests like t9700-perl-git.sh which do most of their - work in an external test script. - - test_external \ - 'GitwebCache::*FileCache*' \ - perl "$TEST_DIRECTORY"/t9503/test_cache_interface.pl - - If the test is outputting its own TAP you should set the - test_external_has_tap variable somewhere before calling the first - test_external* function. See t9700-perl-git.sh for an example. - - # The external test will outputs its own plan - test_external_has_tap=1 - - - test_external_without_stderr [<prereq>] <message> <external> <script> - - Like test_external but fail if there's any output on stderr, - instead of checking the exit code. - - test_external_without_stderr \ - 'Perl API' \ - perl "$TEST_DIRECTORY"/t9700/test.pl - - test_expect_code <exit-code> <command> Run a command and ensure that it exits with the given exit code. diff --git a/t/helper/test-fast-rebase.c b/t/helper/test-fast-rebase.c index 4e5553e202..45665ec19a 100644 --- a/t/helper/test-fast-rebase.c +++ b/t/helper/test-fast-rebase.c @@ -184,8 +184,6 @@ int cmd__fast_rebase(int argc, const char **argv) last_picked_commit = commit; last_commit = create_commit(result.tree, commit, last_commit); } - /* TODO: There should be some kind of rev_info_free(&revs) call... */ - memset(&revs, 0, sizeof(revs)); merge_switch_to_result(&merge_opt, head_tree, &result, 1, !result.clean); diff --git a/t/helper/test-mergesort.c b/t/helper/test-mergesort.c index ebf68f7de8..202e54a7ff 100644 --- a/t/helper/test-mergesort.c +++ b/t/helper/test-mergesort.c @@ -13,19 +13,10 @@ struct line { struct line *next; }; -static void *get_next(const void *a) -{ - return ((const struct line *)a)->next; -} - -static void set_next(void *a, void *b) -{ - ((struct line *)a)->next = b; -} +DEFINE_LIST_SORT(static, sort_lines, struct line, next); -static int compare_strings(const void *a, const void *b) +static int compare_strings(const struct line *x, const struct line *y) { - const struct line *x = a, *y = b; return strcmp(x->text, y->text); } @@ -47,7 +38,7 @@ static int sort_stdin(void) p = line; } - lines = llist_mergesort(lines, get_next, set_next, compare_strings); + sort_lines(&lines, compare_strings); while (lines) { puts(lines->text); @@ -273,21 +264,11 @@ struct number { struct number *next; }; -static void *get_next_number(const void *a) -{ - stats.get_next++; - return ((const struct number *)a)->next; -} - -static void set_next_number(void *a, void *b) -{ - stats.set_next++; - ((struct number *)a)->next = b; -} +DEFINE_LIST_SORT_DEBUG(static, sort_numbers, struct number, next, + stats.get_next++, stats.set_next++); -static int compare_numbers(const void *av, const void *bv) +static int compare_numbers(const struct number *an, const struct number *bn) { - const struct number *an = av, *bn = bv; int a = an->value, b = bn->value; stats.compare++; return (a > b) - (a < b); @@ -325,8 +306,7 @@ static int test(const struct dist *dist, const struct mode *mode, int n, int m) *tail = NULL; stats.get_next = stats.set_next = stats.compare = 0; - list = llist_mergesort(list, get_next_number, set_next_number, - compare_numbers); + sort_numbers(&list, compare_numbers); QSORT(arr, n, compare_ints); for (i = 0, curr = list; i < n && curr; i++, curr = curr->next) { diff --git a/t/helper/test-rot13-filter.c b/t/helper/test-rot13-filter.c new file mode 100644 index 0000000000..f8d564c622 --- /dev/null +++ b/t/helper/test-rot13-filter.c @@ -0,0 +1,382 @@ +/* + * Example implementation for the Git filter protocol version 2 + * See Documentation/gitattributes.txt, section "Filter Protocol" + * + * Usage: test-tool rot13-filter [--always-delay] --log=<path> <capabilities> + * + * Log path defines a debug log file that the script writes to. The + * subsequent arguments define a list of supported protocol capabilities + * ("clean", "smudge", etc). + * + * When --always-delay is given all pathnames with the "can-delay" flag + * that don't appear on the list bellow are delayed with a count of 1 + * (see more below). + * + * This implementation supports special test cases: + * (1) If data with the pathname "clean-write-fail.r" is processed with + * a "clean" operation then the write operation will die. + * (2) If data with the pathname "smudge-write-fail.r" is processed with + * a "smudge" operation then the write operation will die. + * (3) If data with the pathname "error.r" is processed with any + * operation then the filter signals that it cannot or does not want + * to process the file. + * (4) If data with the pathname "abort.r" is processed with any + * operation then the filter signals that it cannot or does not want + * to process the file and any file after that is processed with the + * same command. + * (5) If data with a pathname that is a key in the delay hash is + * requested (e.g. "test-delay10.a") then the filter responds with + * a "delay" status and sets the "requested" field in the delay hash. + * The filter will signal the availability of this object after + * "count" (field in delay hash) "list_available_blobs" commands. + * (6) If data with the pathname "missing-delay.a" is processed that the + * filter will drop the path from the "list_available_blobs" response. + * (7) If data with the pathname "invalid-delay.a" is processed that the + * filter will add the path "unfiltered" which was not delayed before + * to the "list_available_blobs" response. + */ + +#include "test-tool.h" +#include "pkt-line.h" +#include "string-list.h" +#include "strmap.h" +#include "parse-options.h" + +static FILE *logfile; +static int always_delay, has_clean_cap, has_smudge_cap; +static struct strmap delay = STRMAP_INIT; + +static inline const char *str_or_null(const char *str) +{ + return str ? str : "(null)"; +} + +static char *rot13(char *str) +{ + char *c; + for (c = str; *c; c++) + if (isalpha(*c)) + *c += tolower(*c) < 'n' ? 13 : -13; + return str; +} + +static char *get_value(char *buf, const char *key) +{ + const char *orig_buf = buf; + if (!buf || + !skip_prefix((const char *)buf, key, (const char **)&buf) || + !skip_prefix((const char *)buf, "=", (const char **)&buf) || + !*buf) + die("expected key '%s', got '%s'", key, str_or_null(orig_buf)); + return buf; +} + +/* + * Read a text packet, expecting that it is in the form "key=value" for + * the given key. An EOF does not trigger any error and is reported + * back to the caller with NULL. Die if the "key" part of "key=value" does + * not match the given key, or the value part is empty. + */ +static char *packet_key_val_read(const char *key) +{ + char *buf; + if (packet_read_line_gently(0, NULL, &buf) < 0) + return NULL; + return xstrdup(get_value(buf, key)); +} + +static inline void assert_remote_capability(struct strset *caps, const char *cap) +{ + if (!strset_contains(caps, cap)) + die("required '%s' capability not available from remote", cap); +} + +static void read_capabilities(struct strset *remote_caps) +{ + for (;;) { + char *buf = packet_read_line(0, NULL); + if (!buf) + break; + strset_add(remote_caps, get_value(buf, "capability")); + } + + assert_remote_capability(remote_caps, "clean"); + assert_remote_capability(remote_caps, "smudge"); + assert_remote_capability(remote_caps, "delay"); +} + +static void check_and_write_capabilities(struct strset *remote_caps, + const char **caps, int nr_caps) +{ + int i; + for (i = 0; i < nr_caps; i++) { + if (!strset_contains(remote_caps, caps[i])) + die("our capability '%s' is not available from remote", + caps[i]); + packet_write_fmt(1, "capability=%s\n", caps[i]); + } + packet_flush(1); +} + +struct delay_entry { + int requested, count; + char *output; +}; + +static void free_delay_entries(void) +{ + struct hashmap_iter iter; + struct strmap_entry *ent; + + strmap_for_each_entry(&delay, &iter, ent) { + struct delay_entry *delay_entry = ent->value; + free(delay_entry->output); + free(delay_entry); + } + strmap_clear(&delay, 0); +} + +static void add_delay_entry(char *pathname, int count, int requested) +{ + struct delay_entry *entry = xcalloc(1, sizeof(*entry)); + entry->count = count; + entry->requested = requested; + if (strmap_put(&delay, pathname, entry)) + BUG("adding the same path twice to delay hash?"); +} + +static void reply_list_available_blobs_cmd(void) +{ + struct hashmap_iter iter; + struct strmap_entry *ent; + struct string_list_item *str_item; + struct string_list paths = STRING_LIST_INIT_NODUP; + + /* flush */ + if (packet_read_line(0, NULL)) + die("bad list_available_blobs end"); + + strmap_for_each_entry(&delay, &iter, ent) { + struct delay_entry *delay_entry = ent->value; + if (!delay_entry->requested) + continue; + delay_entry->count--; + if (!strcmp(ent->key, "invalid-delay.a")) { + /* Send Git a pathname that was not delayed earlier */ + packet_write_fmt(1, "pathname=unfiltered"); + } + if (!strcmp(ent->key, "missing-delay.a")) { + /* Do not signal Git that this file is available */ + } else if (!delay_entry->count) { + string_list_append(&paths, ent->key); + packet_write_fmt(1, "pathname=%s", ent->key); + } + } + + /* Print paths in sorted order. */ + string_list_sort(&paths); + for_each_string_list_item(str_item, &paths) + fprintf(logfile, " %s", str_item->string); + string_list_clear(&paths, 0); + + packet_flush(1); + + fprintf(logfile, " [OK]\n"); + packet_write_fmt(1, "status=success"); + packet_flush(1); +} + +static void command_loop(void) +{ + for (;;) { + char *buf, *output; + char *pathname; + struct delay_entry *entry; + struct strbuf input = STRBUF_INIT; + char *command = packet_key_val_read("command"); + + if (!command) { + fprintf(logfile, "STOP\n"); + break; + } + fprintf(logfile, "IN: %s", command); + + if (!strcmp(command, "list_available_blobs")) { + reply_list_available_blobs_cmd(); + free(command); + continue; + } + + pathname = packet_key_val_read("pathname"); + if (!pathname) + die("unexpected EOF while expecting pathname"); + fprintf(logfile, " %s", pathname); + + /* Read until flush */ + while ((buf = packet_read_line(0, NULL))) { + if (!strcmp(buf, "can-delay=1")) { + entry = strmap_get(&delay, pathname); + if (entry && !entry->requested) + entry->requested = 1; + else if (!entry && always_delay) + add_delay_entry(pathname, 1, 1); + } else if (starts_with(buf, "ref=") || + starts_with(buf, "treeish=") || + starts_with(buf, "blob=")) { + fprintf(logfile, " %s", buf); + } else { + /* + * In general, filters need to be graceful about + * new metadata, since it's documented that we + * can pass any key-value pairs, but for tests, + * let's be a little stricter. + */ + die("Unknown message '%s'", buf); + } + } + + read_packetized_to_strbuf(0, &input, 0); + fprintf(logfile, " %"PRIuMAX" [OK] -- ", (uintmax_t)input.len); + + entry = strmap_get(&delay, pathname); + if (entry && entry->output) { + output = entry->output; + } else if (!strcmp(pathname, "error.r") || !strcmp(pathname, "abort.r")) { + output = ""; + } else if (!strcmp(command, "clean") && has_clean_cap) { + output = rot13(input.buf); + } else if (!strcmp(command, "smudge") && has_smudge_cap) { + output = rot13(input.buf); + } else { + die("bad command '%s'", command); + } + + if (!strcmp(pathname, "error.r")) { + fprintf(logfile, "[ERROR]\n"); + packet_write_fmt(1, "status=error"); + packet_flush(1); + } else if (!strcmp(pathname, "abort.r")) { + fprintf(logfile, "[ABORT]\n"); + packet_write_fmt(1, "status=abort"); + packet_flush(1); + } else if (!strcmp(command, "smudge") && + (entry = strmap_get(&delay, pathname)) && + entry->requested == 1) { + fprintf(logfile, "[DELAYED]\n"); + packet_write_fmt(1, "status=delayed"); + packet_flush(1); + entry->requested = 2; + if (entry->output != output) { + free(entry->output); + entry->output = xstrdup(output); + } + } else { + int i, nr_packets = 0; + size_t output_len; + const char *p; + packet_write_fmt(1, "status=success"); + packet_flush(1); + + if (skip_prefix(pathname, command, &p) && + !strcmp(p, "-write-fail.r")) { + fprintf(logfile, "[WRITE FAIL]\n"); + die("%s write error", command); + } + + output_len = strlen(output); + fprintf(logfile, "OUT: %"PRIuMAX" ", (uintmax_t)output_len); + + if (write_packetized_from_buf_no_flush_count(output, + output_len, 1, &nr_packets)) + die("failed to write buffer to stdout"); + packet_flush(1); + + for (i = 0; i < nr_packets; i++) + fprintf(logfile, "."); + fprintf(logfile, " [OK]\n"); + + packet_flush(1); + } + free(pathname); + strbuf_release(&input); + free(command); + } +} + +static void packet_initialize(void) +{ + char *pkt_buf = packet_read_line(0, NULL); + + if (!pkt_buf || strcmp(pkt_buf, "git-filter-client")) + die("bad initialize: '%s'", str_or_null(pkt_buf)); + + pkt_buf = packet_read_line(0, NULL); + if (!pkt_buf || strcmp(pkt_buf, "version=2")) + die("bad version: '%s'", str_or_null(pkt_buf)); + + pkt_buf = packet_read_line(0, NULL); + if (pkt_buf) + die("bad version end: '%s'", pkt_buf); + + packet_write_fmt(1, "git-filter-server"); + packet_write_fmt(1, "version=2"); + packet_flush(1); +} + +static const char *rot13_usage[] = { + "test-tool rot13-filter [--always-delay] --log=<path> <capabilities>", + NULL +}; + +int cmd__rot13_filter(int argc, const char **argv) +{ + int i, nr_caps; + struct strset remote_caps = STRSET_INIT; + const char *log_path = NULL; + + struct option options[] = { + OPT_BOOL(0, "always-delay", &always_delay, + "delay all paths with the can-delay flag"), + OPT_STRING(0, "log", &log_path, "path", + "path to the debug log file"), + OPT_END() + }; + nr_caps = parse_options(argc, argv, NULL, options, rot13_usage, + PARSE_OPT_STOP_AT_NON_OPTION); + + if (!log_path || !nr_caps) + usage_with_options(rot13_usage, options); + + logfile = fopen(log_path, "a"); + if (!logfile) + die_errno("failed to open log file"); + + for (i = 0; i < nr_caps; i++) { + if (!strcmp(argv[i], "smudge")) + has_smudge_cap = 1; + if (!strcmp(argv[i], "clean")) + has_clean_cap = 1; + } + + add_delay_entry("test-delay10.a", 1, 0); + add_delay_entry("test-delay11.a", 1, 0); + add_delay_entry("test-delay20.a", 2, 0); + add_delay_entry("test-delay10.b", 1, 0); + add_delay_entry("missing-delay.a", 1, 0); + add_delay_entry("invalid-delay.a", 1, 0); + + fprintf(logfile, "START\n"); + packet_initialize(); + + read_capabilities(&remote_caps); + check_and_write_capabilities(&remote_caps, argv, nr_caps); + fprintf(logfile, "init handshake complete\n"); + strset_clear(&remote_caps); + + command_loop(); + + if (fclose(logfile)) + die_errno("error closing logfile"); + free_delay_entries(); + return 0; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 49b30057f6..8005588679 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -67,6 +67,7 @@ static struct test_cmd cmds[] = { { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, { "reftable", cmd__reftable }, + { "rot13-filter", cmd__rot13_filter }, { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 487d84da60..a432cc77d9 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -56,6 +56,7 @@ int cmd__read_cache(int argc, const char **argv); int cmd__read_graph(int argc, const char **argv); int cmd__read_midx(int argc, const char **argv); int cmd__ref_store(int argc, const char **argv); +int cmd__rot13_filter(int argc, const char **argv); int cmd__reftable(int argc, const char **argv); int cmd__regex(int argc, const char **argv); int cmd__repository(int argc, const char **argv); diff --git a/t/lib-parallel-checkout.sh b/t/lib-parallel-checkout.sh index 83b279a846..acaee9cbb6 100644 --- a/t/lib-parallel-checkout.sh +++ b/t/lib-parallel-checkout.sh @@ -25,7 +25,11 @@ test_checkout_workers () { local trace_file=trace-test-checkout-workers && rm -f "$trace_file" && - GIT_TRACE2="$(pwd)/$trace_file" "$@" 2>&8 && + ( + GIT_TRACE2="$(pwd)/$trace_file" && + export GIT_TRACE2 && + "$@" 2>&8 + ) && local workers="$(grep "child_start\[..*\] git checkout--worker" "$trace_file" | wc -l)" && test $workers -eq $expected_workers && diff --git a/t/lib-perl.sh b/t/lib-perl.sh new file mode 100644 index 0000000000..d0bf509a16 --- /dev/null +++ b/t/lib-perl.sh @@ -0,0 +1,19 @@ +# Copyright (c) 2022 Ævar Arnfjörð Bjarmason + +test_lazy_prereq PERL_TEST_MORE ' + perl -MTest::More -e 0 +' + +skip_all_if_no_Test_More () { + if ! test_have_prereq PERL + then + skip_all='skipping perl interface tests, perl not available' + test_done + fi + + if ! test_have_prereq PERL_TEST_MORE + then + skip_all="Perl Test::More unavailable, skipping test" + test_done + fi +} diff --git a/t/lib-proto-disable.sh b/t/lib-proto-disable.sh index 83babe57d9..890622be81 100644 --- a/t/lib-proto-disable.sh +++ b/t/lib-proto-disable.sh @@ -1,7 +1,7 @@ # Test routines for checking protocol disabling. -# Test clone/fetch/push with GIT_ALLOW_PROTOCOL whitelist -test_whitelist () { +# Test clone/fetch/push with GIT_ALLOW_PROTOCOL environment variable +test_allow_var () { desc=$1 proto=$2 url=$3 @@ -183,7 +183,7 @@ test_config () { # $2 - machine-readable name of the protocol # $3 - the URL to try cloning test_proto () { - test_whitelist "$@" + test_allow_var "$@" test_config "$@" } diff --git a/t/lib-rebase.sh b/t/lib-rebase.sh index ec6b9b107d..b57541356b 100644 --- a/t/lib-rebase.sh +++ b/t/lib-rebase.sh @@ -207,3 +207,18 @@ check_reworded_commits () { >reword-log && test_cmp reword-expected reword-log } + +# usage: set_replace_editor <file> +# +# Replace the todo file with the exact contents of the given file. +set_replace_editor () { + cat >script <<-\EOF && + cat FILENAME >"$1" + + echo 'rebase -i script after editing:' + cat "$1" + EOF + + sed -e "s/FILENAME/$1/g" <script | write_script fake-editor.sh && + test_set_editor "$(pwd)/fake-editor.sh" +} diff --git a/t/perf/p0004-lazy-init-name-hash.sh b/t/perf/p0004-lazy-init-name-hash.sh index 1afc08fe7f..85be14e4dd 100755 --- a/t/perf/p0004-lazy-init-name-hash.sh +++ b/t/perf/p0004-lazy-init-name-hash.sh @@ -49,7 +49,7 @@ test_perf "single-threaded, $desc" " test-tool lazy-init-name-hash --single --count=$count " -test_perf REPO_BIG_ENOUGH_FOR_MULTI "multi-threaded, $desc" " +test_perf "multi-threaded, $desc" --prereq REPO_BIG_ENOUGH_FOR_MULTI " test-tool lazy-init-name-hash --multi --count=$count " diff --git a/t/perf/p0006-read-tree-checkout.sh b/t/perf/p0006-read-tree-checkout.sh index 900b385c4b..c481c012d2 100755 --- a/t/perf/p0006-read-tree-checkout.sh +++ b/t/perf/p0006-read-tree-checkout.sh @@ -46,7 +46,7 @@ test_expect_success "setup repo" ' ' test_perf "read-tree br_base br_ballast ($nr_files)" ' - git read-tree -m br_base br_ballast -n + git read-tree -n -m br_base br_ballast ' test_perf "switch between br_base br_ballast ($nr_files)" ' diff --git a/t/perf/p0071-sort.sh b/t/perf/p0071-sort.sh index ed366e2e12..ae4ddac864 100755 --- a/t/perf/p0071-sort.sh +++ b/t/perf/p0071-sort.sh @@ -40,11 +40,11 @@ done for file in unsorted sorted reversed do - test_perf "llist_mergesort() $file" " + test_perf "DEFINE_LIST_SORT $file" " test-tool mergesort sort <$file >actual " - test_expect_success "llist_mergesort() $file sorts like sort(1)" " + test_expect_success "DEFINE_LIST_SORT $file sorts like sort(1)" " test_cmp_bin sorted actual " done diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index c181110a43..fce8151d41 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -123,5 +123,6 @@ test_perf_on_all git blame $SPARSE_CONE/f3/a test_perf_on_all git read-tree -mu HEAD test_perf_on_all git checkout-index -f --all test_perf_on_all git update-index --add --remove $SPARSE_CONE/a +test_perf_on_all "git rm -f $SPARSE_CONE/a && git checkout HEAD -- $SPARSE_CONE/a" test_done diff --git a/t/perf/p7527-builtin-fsmonitor.sh b/t/perf/p7527-builtin-fsmonitor.sh index 9338b9ea00..c3f9a4caa4 100755 --- a/t/perf/p7527-builtin-fsmonitor.sh +++ b/t/perf/p7527-builtin-fsmonitor.sh @@ -249,7 +249,7 @@ test_expect_success "Cleanup temp and matrix branches" " do for fsm_val in $fsm_values do - cleanup $uc_val $fsm_val + cleanup $uc_val $fsm_val || return 1 done done " diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 17a268ccd1..502b4bcf9e 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -578,6 +578,78 @@ test_expect_success 'subtest: --run invalid range end' ' EOF_ERR ' +test_expect_success 'subtest: --invert-exit-code without --immediate' ' + run_sub_test_lib_test_err full-pass \ + --invert-exit-code && + check_sub_test_lib_test_err full-pass \ + <<-\EOF_OUT 3<<-EOF_ERR + ok 1 - passing test #1 + ok 2 - passing test #2 + ok 3 - passing test #3 + # passed all 3 test(s) + 1..3 + # faking up non-zero exit with --invert-exit-code + EOF_OUT + EOF_ERR +' + +test_expect_success 'subtest: --invert-exit-code with --immediate: all passed' ' + run_sub_test_lib_test_err full-pass \ + --invert-exit-code --immediate && + check_sub_test_lib_test_err full-pass \ + <<-\EOF_OUT 3<<-EOF_ERR + ok 1 - passing test #1 + ok 2 - passing test #2 + ok 3 - passing test #3 + # passed all 3 test(s) + 1..3 + # faking up non-zero exit with --invert-exit-code + EOF_OUT + EOF_ERR +' + +test_expect_success 'subtest: --invert-exit-code without --immediate: partial pass' ' + run_sub_test_lib_test partial-pass \ + --invert-exit-code && + check_sub_test_lib_test partial-pass <<-\EOF + ok 1 - passing test #1 + not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2 + # false + ok 3 - passing test #3 + # failed 1 among 3 test(s) + 1..3 + # faked up failures as TODO & now exiting with 0 due to --invert-exit-code + EOF +' + +test_expect_success 'subtest: --invert-exit-code with --immediate: partial pass' ' + run_sub_test_lib_test partial-pass \ + --invert-exit-code --immediate && + check_sub_test_lib_test partial-pass \ + <<-\EOF_OUT 3<<-EOF_ERR + ok 1 - passing test #1 + not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2 + # false + 1..2 + # faked up failures as TODO & now exiting with 0 due to --invert-exit-code + EOF_OUT + EOF_ERR +' + +test_expect_success 'subtest: --invert-exit-code --immediate: got a failure' ' + run_sub_test_lib_test partial-pass \ + --invert-exit-code --immediate && + check_sub_test_lib_test_err partial-pass \ + <<-\EOF_OUT 3<<-EOF_ERR + ok 1 - passing test #1 + not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2 + # false + 1..2 + # faked up failures as TODO & now exiting with 0 due to --invert-exit-code + EOF_OUT + EOF_ERR +' + test_expect_success 'subtest: tests respect prerequisites' ' write_and_run_sub_test_lib_test prereqs <<-\EOF && diff --git a/t/t0002-gitfile.sh b/t/t0002-gitfile.sh index f6356db183..26eaca095a 100755 --- a/t/t0002-gitfile.sh +++ b/t/t0002-gitfile.sh @@ -65,7 +65,7 @@ test_expect_success 'check commit-tree' ' test_path_is_file "$REAL/objects/$(objpath $SHA)" ' -test_expect_success !SANITIZE_LEAK 'check rev-list' ' +test_expect_success 'check rev-list' ' git update-ref "HEAD" "$SHA" && git rev-list HEAD >actual && echo $SHA >expected && diff --git a/t/t0004-unwritable.sh b/t/t0004-unwritable.sh index 2e9d652d82..8114fac73b 100755 --- a/t/t0004-unwritable.sh +++ b/t/t0004-unwritable.sh @@ -31,7 +31,7 @@ test_expect_success WRITE_TREE_OUT 'write-tree output on unwritable repository' test_cmp expect out.write-tree ' -test_expect_success POSIXPERM,SANITY,!SANITIZE_LEAK 'commit should notice unwritable repository' ' +test_expect_success POSIXPERM,SANITY 'commit should notice unwritable repository' ' test_when_finished "chmod 775 .git/objects .git/objects/??" && chmod a-w .git/objects .git/objects/?? && test_must_fail git commit -m second 2>out.commit diff --git a/t/t0012-help.sh b/t/t0012-help.sh index 6c33a43690..4ed2f242eb 100755 --- a/t/t0012-help.sh +++ b/t/t0012-help.sh @@ -44,6 +44,8 @@ test_expect_success 'invalid usage' ' test_expect_code 129 git help -g add && test_expect_code 129 git help -a -g && + test_expect_code 129 git help --user-interfaces add && + test_expect_code 129 git help -g -c && test_expect_code 129 git help --config-for-completion add && test_expect_code 129 git help --config-sections-for-completion add @@ -104,9 +106,9 @@ test_expect_success 'git help' ' test_i18ngrep "^ commit " help.output && test_i18ngrep "^ fetch " help.output ' + test_expect_success 'git help -g' ' git help -g >help.output && - test_i18ngrep "^ attributes " help.output && test_i18ngrep "^ everyday " help.output && test_i18ngrep "^ tutorial " help.output ' @@ -127,6 +129,12 @@ test_expect_success 'git help succeeds without git.html' ' test_cmp expect test-browser.log ' +test_expect_success 'git help --user-interfaces' ' + git help --user-interfaces >help.output && + grep "^ attributes " help.output && + grep "^ mailmap " help.output +' + test_expect_success 'git help -c' ' git help -c >help.output && cat >expect <<-\EOF && @@ -220,6 +228,10 @@ test_expect_success "'git help -a' section spacing" ' Low-level Commands / Syncing Repositories Low-level Commands / Internal Helpers + + User-facing repository, command and file interfaces + + Developer-facing file file formats, protocols and interfaces EOF test_cmp expect actual ' diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index bad37abad2..abecd75e4e 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -17,9 +17,6 @@ tr \ 'nopqrstuvwxyzabcdefghijklmNOPQRSTUVWXYZABCDEFGHIJKLM' EOF -write_script rot13-filter.pl "$PERL_PATH" \ - <"$TEST_DIRECTORY"/t0021/rot13-filter.pl - generate_random_characters () { LEN=$1 NAME=$2 @@ -365,8 +362,8 @@ test_expect_success 'diff does not reuse worktree files that need cleaning' ' test_line_count = 0 count ' -test_expect_success PERL 'required process filter should filter data' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'required process filter should filter data' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && test_config_global filter.protocol.required true && rm -rf repo && mkdir repo && @@ -450,8 +447,8 @@ test_expect_success PERL 'required process filter should filter data' ' ) ' -test_expect_success PERL 'required process filter should filter data for various subcommands' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'required process filter should filter data for various subcommands' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && test_config_global filter.protocol.required true && ( cd repo && @@ -561,9 +558,9 @@ test_expect_success PERL 'required process filter should filter data for various ) ' -test_expect_success PERL 'required process filter takes precedence' ' +test_expect_success 'required process filter takes precedence' ' test_config_global filter.protocol.clean false && - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean" && + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean" && test_config_global filter.protocol.required true && rm -rf repo && mkdir repo && @@ -587,8 +584,8 @@ test_expect_success PERL 'required process filter takes precedence' ' ) ' -test_expect_success PERL 'required process filter should be used only for "clean" operation only' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean" && +test_expect_success 'required process filter should be used only for "clean" operation only' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean" && rm -rf repo && mkdir repo && ( @@ -622,8 +619,8 @@ test_expect_success PERL 'required process filter should be used only for "clean ) ' -test_expect_success PERL 'required process filter should process multiple packets' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'required process filter should process multiple packets' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && test_config_global filter.protocol.required true && rm -rf repo && @@ -687,8 +684,8 @@ test_expect_success PERL 'required process filter should process multiple packet ) ' -test_expect_success PERL 'required process filter with clean error should fail' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'required process filter with clean error should fail' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && test_config_global filter.protocol.required true && rm -rf repo && mkdir repo && @@ -706,8 +703,8 @@ test_expect_success PERL 'required process filter with clean error should fail' ) ' -test_expect_success PERL 'process filter should restart after unexpected write failure' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'process filter should restart after unexpected write failure' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && rm -rf repo && mkdir repo && ( @@ -735,7 +732,7 @@ test_expect_success PERL 'process filter should restart after unexpected write f rm -f debug.log && git checkout --quiet --no-progress . 2>git-stderr.log && - grep "smudge write error at" git-stderr.log && + grep "smudge write error" git-stderr.log && test_i18ngrep "error: external filter" git-stderr.log && cat >expected.log <<-EOF && @@ -761,8 +758,8 @@ test_expect_success PERL 'process filter should restart after unexpected write f ) ' -test_expect_success PERL 'process filter should not be restarted if it signals an error' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'process filter should not be restarted if it signals an error' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && rm -rf repo && mkdir repo && ( @@ -804,8 +801,8 @@ test_expect_success PERL 'process filter should not be restarted if it signals a ) ' -test_expect_success PERL 'process filter abort stops processing of all further files' ' - test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" && +test_expect_success 'process filter abort stops processing of all further files' ' + test_config_global filter.protocol.process "test-tool rot13-filter --log=debug.log clean smudge" && rm -rf repo && mkdir repo && ( @@ -861,10 +858,10 @@ test_expect_success PERL 'invalid process filter must fail (and not hang!)' ' ) ' -test_expect_success PERL 'delayed checkout in process filter' ' - test_config_global filter.a.process "rot13-filter.pl a.log clean smudge delay" && +test_expect_success 'delayed checkout in process filter' ' + test_config_global filter.a.process "test-tool rot13-filter --log=a.log clean smudge delay" && test_config_global filter.a.required true && - test_config_global filter.b.process "rot13-filter.pl b.log clean smudge delay" && + test_config_global filter.b.process "test-tool rot13-filter --log=b.log clean smudge delay" && test_config_global filter.b.required true && rm -rf repo && @@ -940,8 +937,8 @@ test_expect_success PERL 'delayed checkout in process filter' ' ) ' -test_expect_success PERL 'missing file in delayed checkout' ' - test_config_global filter.bug.process "rot13-filter.pl bug.log clean smudge delay" && +test_expect_success 'missing file in delayed checkout' ' + test_config_global filter.bug.process "test-tool rot13-filter --log=bug.log clean smudge delay" && test_config_global filter.bug.required true && rm -rf repo && @@ -960,8 +957,8 @@ test_expect_success PERL 'missing file in delayed checkout' ' grep "error: .missing-delay\.a. was not filtered properly" git-stderr.log ' -test_expect_success PERL 'invalid file in delayed checkout' ' - test_config_global filter.bug.process "rot13-filter.pl bug.log clean smudge delay" && +test_expect_success 'invalid file in delayed checkout' ' + test_config_global filter.bug.process "test-tool rot13-filter --log=bug.log clean smudge delay" && test_config_global filter.bug.required true && rm -rf repo && @@ -990,10 +987,10 @@ do mode_prereq='UTF8_NFD_TO_NFC' ;; esac - test_expect_success PERL,SYMLINKS,$mode_prereq \ + test_expect_success SYMLINKS,$mode_prereq \ "delayed checkout with $mode-collision don't write to the wrong place" ' test_config_global filter.delay.process \ - "\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" && + "test-tool rot13-filter --always-delay --log=delayed.log clean smudge delay" && test_config_global filter.delay.required true && git init $mode-collision && @@ -1026,12 +1023,12 @@ do ' done -test_expect_success PERL,SYMLINKS,CASE_INSENSITIVE_FS \ +test_expect_success SYMLINKS,CASE_INSENSITIVE_FS \ "delayed checkout with submodule collision don't write to the wrong place" ' git init collision-with-submodule && ( cd collision-with-submodule && - git config filter.delay.process "\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" && + git config filter.delay.process "test-tool rot13-filter --always-delay --log=delayed.log clean smudge delay" && git config filter.delay.required true && # We need Git to treat the submodule "a" and the @@ -1062,11 +1059,11 @@ test_expect_success PERL,SYMLINKS,CASE_INSENSITIVE_FS \ ) ' -test_expect_success PERL 'setup for progress tests' ' +test_expect_success 'setup for progress tests' ' git init progress && ( cd progress && - git config filter.delay.process "rot13-filter.pl delay-progress.log clean smudge delay" && + git config filter.delay.process "test-tool rot13-filter --log=delay-progress.log clean smudge delay" && git config filter.delay.required true && echo "*.a filter=delay" >.gitattributes && @@ -1132,4 +1129,26 @@ do ' done +test_expect_success 'delayed checkout correctly reports the number of updated entries' ' + rm -rf repo && + git init repo && + ( + cd repo && + git config filter.delay.process "test-tool rot13-filter --log=delayed.log clean smudge delay" && + git config filter.delay.required true && + + echo "*.a filter=delay" >.gitattributes && + echo a >test-delay10.a && + echo a >test-delay11.a && + git add . && + git commit -m files && + + rm *.a && + git checkout . 2>err && + grep "IN: smudge test-delay10.a .* \\[DELAYED\\]" delayed.log && + grep "IN: smudge test-delay11.a .* \\[DELAYED\\]" delayed.log && + grep "Updated 2 paths from the index" err + ) +' + test_done diff --git a/t/t0021/rot13-filter.pl b/t/t0021/rot13-filter.pl deleted file mode 100644 index 7bb93768f3..0000000000 --- a/t/t0021/rot13-filter.pl +++ /dev/null @@ -1,247 +0,0 @@ -# -# Example implementation for the Git filter protocol version 2 -# See Documentation/gitattributes.txt, section "Filter Protocol" -# -# Usage: rot13-filter.pl [--always-delay] <log path> <capabilities> -# -# Log path defines a debug log file that the script writes to. The -# subsequent arguments define a list of supported protocol capabilities -# ("clean", "smudge", etc). -# -# When --always-delay is given all pathnames with the "can-delay" flag -# that don't appear on the list bellow are delayed with a count of 1 -# (see more below). -# -# This implementation supports special test cases: -# (1) If data with the pathname "clean-write-fail.r" is processed with -# a "clean" operation then the write operation will die. -# (2) If data with the pathname "smudge-write-fail.r" is processed with -# a "smudge" operation then the write operation will die. -# (3) If data with the pathname "error.r" is processed with any -# operation then the filter signals that it cannot or does not want -# to process the file. -# (4) If data with the pathname "abort.r" is processed with any -# operation then the filter signals that it cannot or does not want -# to process the file and any file after that is processed with the -# same command. -# (5) If data with a pathname that is a key in the DELAY hash is -# requested (e.g. "test-delay10.a") then the filter responds with -# a "delay" status and sets the "requested" field in the DELAY hash. -# The filter will signal the availability of this object after -# "count" (field in DELAY hash) "list_available_blobs" commands. -# (6) If data with the pathname "missing-delay.a" is processed that the -# filter will drop the path from the "list_available_blobs" response. -# (7) If data with the pathname "invalid-delay.a" is processed that the -# filter will add the path "unfiltered" which was not delayed before -# to the "list_available_blobs" response. -# - -use 5.008; -sub gitperllib { - # Git assumes that all path lists are Unix-y colon-separated ones. But - # when the Git for Windows executes the test suite, its MSYS2 Bash - # calls git.exe, and colon-separated path lists are converted into - # Windows-y semicolon-separated lists of *Windows* paths (which - # naturally contain a colon after the drive letter, so splitting by - # colons simply does not cut it). - # - # Detect semicolon-separated path list and handle them appropriately. - - if ($ENV{GITPERLLIB} =~ /;/) { - return split(/;/, $ENV{GITPERLLIB}); - } - return split(/:/, $ENV{GITPERLLIB}); -} -use lib (gitperllib()); -use strict; -use warnings; -use IO::File; -use Git::Packet; - -my $MAX_PACKET_CONTENT_SIZE = 65516; - -my $always_delay = 0; -if ( $ARGV[0] eq '--always-delay' ) { - $always_delay = 1; - shift @ARGV; -} - -my $log_file = shift @ARGV; -my @capabilities = @ARGV; - -open my $debug, ">>", $log_file or die "cannot open log file: $!"; - -my %DELAY = ( - 'test-delay10.a' => { "requested" => 0, "count" => 1 }, - 'test-delay11.a' => { "requested" => 0, "count" => 1 }, - 'test-delay20.a' => { "requested" => 0, "count" => 2 }, - 'test-delay10.b' => { "requested" => 0, "count" => 1 }, - 'missing-delay.a' => { "requested" => 0, "count" => 1 }, - 'invalid-delay.a' => { "requested" => 0, "count" => 1 }, -); - -sub rot13 { - my $str = shift; - $str =~ y/A-Za-z/N-ZA-Mn-za-m/; - return $str; -} - -print $debug "START\n"; -$debug->flush(); - -packet_initialize("git-filter", 2); - -my %remote_caps = packet_read_and_check_capabilities("clean", "smudge", "delay"); -packet_check_and_write_capabilities(\%remote_caps, @capabilities); - -print $debug "init handshake complete\n"; -$debug->flush(); - -while (1) { - my ( $res, $command ) = packet_key_val_read("command"); - if ( $res == -1 ) { - print $debug "STOP\n"; - exit(); - } - print $debug "IN: $command"; - $debug->flush(); - - if ( $command eq "list_available_blobs" ) { - # Flush - packet_compare_lists([1, ""], packet_bin_read()) || - die "bad list_available_blobs end"; - - foreach my $pathname ( sort keys %DELAY ) { - if ( $DELAY{$pathname}{"requested"} >= 1 ) { - $DELAY{$pathname}{"count"} = $DELAY{$pathname}{"count"} - 1; - if ( $pathname eq "invalid-delay.a" ) { - # Send Git a pathname that was not delayed earlier - packet_txt_write("pathname=unfiltered"); - } - if ( $pathname eq "missing-delay.a" ) { - # Do not signal Git that this file is available - } elsif ( $DELAY{$pathname}{"count"} == 0 ) { - print $debug " $pathname"; - packet_txt_write("pathname=$pathname"); - } - } - } - - packet_flush(); - - print $debug " [OK]\n"; - $debug->flush(); - packet_txt_write("status=success"); - packet_flush(); - } else { - my ( $res, $pathname ) = packet_key_val_read("pathname"); - if ( $res == -1 ) { - die "unexpected EOF while expecting pathname"; - } - print $debug " $pathname"; - $debug->flush(); - - # Read until flush - my ( $done, $buffer ) = packet_txt_read(); - while ( $buffer ne '' ) { - if ( $buffer eq "can-delay=1" ) { - if ( exists $DELAY{$pathname} and $DELAY{$pathname}{"requested"} == 0 ) { - $DELAY{$pathname}{"requested"} = 1; - } elsif ( !exists $DELAY{$pathname} and $always_delay ) { - $DELAY{$pathname} = { "requested" => 1, "count" => 1 }; - } - } elsif ($buffer =~ /^(ref|treeish|blob)=/) { - print $debug " $buffer"; - } else { - # In general, filters need to be graceful about - # new metadata, since it's documented that we - # can pass any key-value pairs, but for tests, - # let's be a little stricter. - die "Unknown message '$buffer'"; - } - - ( $done, $buffer ) = packet_txt_read(); - } - if ( $done == -1 ) { - die "unexpected EOF after pathname '$pathname'"; - } - - my $input = ""; - { - binmode(STDIN); - my $buffer; - my $done = 0; - while ( !$done ) { - ( $done, $buffer ) = packet_bin_read(); - $input .= $buffer; - } - if ( $done == -1 ) { - die "unexpected EOF while reading input for '$pathname'"; - } - print $debug " " . length($input) . " [OK] -- "; - $debug->flush(); - } - - my $output; - if ( exists $DELAY{$pathname} and exists $DELAY{$pathname}{"output"} ) { - $output = $DELAY{$pathname}{"output"} - } elsif ( $pathname eq "error.r" or $pathname eq "abort.r" ) { - $output = ""; - } elsif ( $command eq "clean" and grep( /^clean$/, @capabilities ) ) { - $output = rot13($input); - } elsif ( $command eq "smudge" and grep( /^smudge$/, @capabilities ) ) { - $output = rot13($input); - } else { - die "bad command '$command'"; - } - - if ( $pathname eq "error.r" ) { - print $debug "[ERROR]\n"; - $debug->flush(); - packet_txt_write("status=error"); - packet_flush(); - } elsif ( $pathname eq "abort.r" ) { - print $debug "[ABORT]\n"; - $debug->flush(); - packet_txt_write("status=abort"); - packet_flush(); - } elsif ( $command eq "smudge" and - exists $DELAY{$pathname} and - $DELAY{$pathname}{"requested"} == 1 ) { - print $debug "[DELAYED]\n"; - $debug->flush(); - packet_txt_write("status=delayed"); - packet_flush(); - $DELAY{$pathname}{"requested"} = 2; - $DELAY{$pathname}{"output"} = $output; - } else { - packet_txt_write("status=success"); - packet_flush(); - - if ( $pathname eq "${command}-write-fail.r" ) { - print $debug "[WRITE FAIL]\n"; - $debug->flush(); - die "${command} write error"; - } - - print $debug "OUT: " . length($output) . " "; - $debug->flush(); - - while ( length($output) > 0 ) { - my $packet = substr( $output, 0, $MAX_PACKET_CONTENT_SIZE ); - packet_bin_write($packet); - # dots represent the number of packets - print $debug "."; - if ( length($output) > $MAX_PACKET_CONTENT_SIZE ) { - $output = substr( $output, $MAX_PACKET_CONTENT_SIZE ); - } else { - $output = ""; - } - } - packet_flush(); - print $debug " [OK]\n"; - $debug->flush(); - packet_flush(); - } - } -} diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 7f80f46393..a22e0e1382 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -2,6 +2,7 @@ test_description='CRLF conversion all combinations' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh compare_files () { diff --git a/t/t0032-reftable-unittest.sh b/t/t0032-reftable-unittest.sh index 0ed14971a5..471cb37ac2 100755 --- a/t/t0032-reftable-unittest.sh +++ b/t/t0032-reftable-unittest.sh @@ -5,6 +5,7 @@ test_description='reftable unittests' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'unittests' ' diff --git a/t/t0033-safe-directory.sh b/t/t0033-safe-directory.sh index f4d737dadd..aecb308cf6 100755 --- a/t/t0033-safe-directory.sh +++ b/t/t0033-safe-directory.sh @@ -2,6 +2,7 @@ test_description='verify safe.directory checks' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh GIT_TEST_ASSUME_DIFFERENT_OWNER=1 diff --git a/t/t0050-filesystem.sh b/t/t0050-filesystem.sh index 5c9dc90d0b..325eb1c3cd 100755 --- a/t/t0050-filesystem.sh +++ b/t/t0050-filesystem.sh @@ -5,6 +5,7 @@ test_description='Various filesystem issues' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh auml=$(printf '\303\244') diff --git a/t/t0071-sort.sh b/t/t0071-sort.sh index 6f9a501c72..ba8ad1d1ca 100755 --- a/t/t0071-sort.sh +++ b/t/t0071-sort.sh @@ -5,7 +5,7 @@ test_description='verify sort functions' TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh -test_expect_success 'llist_mergesort()' ' +test_expect_success 'DEFINE_LIST_SORT_DEBUG' ' test-tool mergesort test ' diff --git a/t/t0091-bugreport.sh b/t/t0091-bugreport.sh index 08f5fe9cae..b6d2f591ac 100755 --- a/t/t0091-bugreport.sh +++ b/t/t0091-bugreport.sh @@ -78,4 +78,52 @@ test_expect_success 'indicates populated hooks' ' test_cmp expect actual ' +test_expect_success UNZIP '--diagnose creates diagnostics zip archive' ' + test_when_finished rm -rf report && + + git bugreport --diagnose -o report -s test >out && + + zip_path=report/git-diagnostics-test.zip && + grep "Available space" out && + test_path_is_file "$zip_path" && + + # Check zipped archive content + "$GIT_UNZIP" -p "$zip_path" diagnostics.log >out && + test_file_not_empty out && + + "$GIT_UNZIP" -p "$zip_path" packs-local.txt >out && + grep ".git/objects" out && + + "$GIT_UNZIP" -p "$zip_path" objects-local.txt >out && + grep "^Total: [0-9][0-9]*" out && + + # Should not include .git directory contents by default + ! "$GIT_UNZIP" -l "$zip_path" | grep ".git/" +' + +test_expect_success UNZIP '--diagnose=stats excludes .git dir contents' ' + test_when_finished rm -rf report && + + git bugreport --diagnose=stats -o report -s test >out && + + # Includes pack quantity/size info + "$GIT_UNZIP" -p "$zip_path" packs-local.txt >out && + grep ".git/objects" out && + + # Does not include .git directory contents + ! "$GIT_UNZIP" -l "$zip_path" | grep ".git/" +' + +test_expect_success UNZIP '--diagnose=all includes .git dir contents' ' + test_when_finished rm -rf report && + + git bugreport --diagnose=all -o report -s test >out && + + # Includes .git directory contents + "$GIT_UNZIP" -l "$zip_path" | grep ".git/" && + + "$GIT_UNZIP" -p "$zip_path" .git/HEAD >out && + test_file_not_empty out +' + test_done diff --git a/t/t0092-diagnose.sh b/t/t0092-diagnose.sh new file mode 100755 index 0000000000..fca9b58489 --- /dev/null +++ b/t/t0092-diagnose.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +test_description='git diagnose' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +test_expect_success UNZIP 'creates diagnostics zip archive' ' + test_when_finished rm -rf report && + + git diagnose -o report -s test >out && + grep "Available space" out && + + zip_path=report/git-diagnostics-test.zip && + test_path_is_file "$zip_path" && + + # Check zipped archive content + "$GIT_UNZIP" -p "$zip_path" diagnostics.log >out && + test_file_not_empty out && + + "$GIT_UNZIP" -p "$zip_path" packs-local.txt >out && + grep ".git/objects" out && + + "$GIT_UNZIP" -p "$zip_path" objects-local.txt >out && + grep "^Total: [0-9][0-9]*" out && + + # Should not include .git directory contents by default + ! "$GIT_UNZIP" -l "$zip_path" | grep ".git/" +' + +test_expect_success UNZIP '--mode=stats excludes .git dir contents' ' + test_when_finished rm -rf report && + + git diagnose -o report -s test --mode=stats >out && + + # Includes pack quantity/size info + "$GIT_UNZIP" -p "$zip_path" packs-local.txt >out && + grep ".git/objects" out && + + # Does not include .git directory contents + ! "$GIT_UNZIP" -l "$zip_path" | grep ".git/" +' + +test_expect_success UNZIP '--mode=all includes .git dir contents' ' + test_when_finished rm -rf report && + + git diagnose -o report -s test --mode=all >out && + + # Includes pack quantity/size info + "$GIT_UNZIP" -p "$zip_path" packs-local.txt >out && + grep ".git/objects" out && + + # Includes .git directory contents + "$GIT_UNZIP" -l "$zip_path" | grep ".git/" && + + "$GIT_UNZIP" -p "$zip_path" .git/HEAD >out && + test_file_not_empty out +' + +test_done diff --git a/t/t0095-bloom.sh b/t/t0095-bloom.sh index daeb4a5e3e..b567383eb8 100755 --- a/t/t0095-bloom.sh +++ b/t/t0095-bloom.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='Testing the various Bloom filter computations in bloom.c' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'compute unseeded murmur3 hash for empty string' ' diff --git a/t/t0202-gettext-perl.sh b/t/t0202-gettext-perl.sh index df2ea34932..5a6f28051b 100755 --- a/t/t0202-gettext-perl.sh +++ b/t/t0202-gettext-perl.sh @@ -7,22 +7,12 @@ test_description='Perl gettext interface (Git::I18N)' TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh +. "$TEST_DIRECTORY"/lib-perl.sh +skip_all_if_no_Test_More -if ! test_have_prereq PERL; then - skip_all='skipping perl interface tests, perl not available' - test_done -fi - -perl -MTest::More -e 0 2>/dev/null || { - skip_all="Perl Test::More unavailable, skipping test" - test_done -} - -# The external test will outputs its own plan -test_external_has_tap=1 - -test_external_without_stderr \ - 'Perl Git::I18N API' \ - perl "$TEST_DIRECTORY"/t0202/test.pl +test_expect_success 'run t0202/test.pl to test Git::I18N.pm' ' + "$PERL_PATH" "$TEST_DIRECTORY"/t0202/test.pl 2>stderr && + test_must_be_empty stderr +' test_done diff --git a/t/t0203-gettext-setlocale-sanity.sh b/t/t0203-gettext-setlocale-sanity.sh index 0ce1f22eff..86cff324ff 100755 --- a/t/t0203-gettext-setlocale-sanity.sh +++ b/t/t0203-gettext-setlocale-sanity.sh @@ -5,6 +5,7 @@ test_description="The Git C functions aren't broken by setlocale(3)" +TEST_PASSES_SANITIZE_LEAK=true . ./lib-gettext.sh test_expect_success 'git show a ISO-8859-1 commit under C locale' ' diff --git a/t/t0212/parse_events.perl b/t/t0212/parse_events.perl index b6408560c0..30a9f51e9f 100644 --- a/t/t0212/parse_events.perl +++ b/t/t0212/parse_events.perl @@ -216,12 +216,19 @@ while (<>) { elsif ($event eq 'data') { my $cat = $line->{'category'}; - if ($cat eq 'test_category') { - - my $key = $line->{'key'}; - my $value = $line->{'value'}; - $processes->{$sid}->{'data'}->{$cat}->{$key} = $value; - } + my $key = $line->{'key'}; + my $value = $line->{'value'}; + $processes->{$sid}->{'data'}->{$cat}->{$key} = $value; + } + + elsif ($event eq 'data_json') { + # NEEDSWORK: Ignore due to + # compat/win32/trace2_win32_process_info.c, which should log a + # "cmd_ancestry" event instead. + } + + else { + push @{$processes->{$sid}->{$event}} => $line->{value}; } # This trace2 target does not emit 'printf' events. diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index dadf3b1458..23b8942edb 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -88,7 +88,8 @@ done for opt in --buffer \ --follow-symlinks \ - --batch-all-objects + --batch-all-objects \ + -z do test_expect_success "usage: bad option combination: $opt without batch mode" ' test_incompatible_usage git cat-file $opt && @@ -100,6 +101,10 @@ echo_without_newline () { printf '%s' "$*" } +echo_without_newline_nul () { + echo_without_newline "$@" | tr '\n' '\0' +} + strlen () { echo_without_newline "$1" | wc -c | sed -e 's/^ *//' } @@ -398,6 +403,12 @@ test_expect_success '--batch with multiple sha1s gives correct format' ' test "$(maybe_remove_timestamp "$batch_output" 1)" = "$(maybe_remove_timestamp "$(echo_without_newline "$batch_input" | git cat-file --batch)" 1)" ' +test_expect_success '--batch, -z with multiple sha1s gives correct format' ' + echo_without_newline_nul "$batch_input" >in && + test "$(maybe_remove_timestamp "$batch_output" 1)" = \ + "$(maybe_remove_timestamp "$(git cat-file --batch -z <in)" 1)" +' + batch_check_input="$hello_sha1 $tree_sha1 $commit_sha1 @@ -418,6 +429,30 @@ test_expect_success "--batch-check with multiple sha1s gives correct format" ' "$(echo_without_newline "$batch_check_input" | git cat-file --batch-check)" ' +test_expect_success "--batch-check, -z with multiple sha1s gives correct format" ' + echo_without_newline_nul "$batch_check_input" >in && + test "$batch_check_output" = "$(git cat-file --batch-check -z <in)" +' + +test_expect_success FUNNYNAMES '--batch-check, -z with newline in input' ' + touch -- "newline${LF}embedded" && + git add -- "newline${LF}embedded" && + git commit -m "file with newline embedded" && + test_tick && + + printf "HEAD:newline${LF}embedded" >in && + git cat-file --batch-check -z <in >actual && + + echo "$(git rev-parse "HEAD:newline${LF}embedded") blob 0" >expect && + test_cmp expect actual +' + +batch_command_multiple_info="info $hello_sha1 +info $tree_sha1 +info $commit_sha1 +info $tag_sha1 +info deadbeef" + test_expect_success '--batch-command with multiple info calls gives correct format' ' cat >expect <<-EOF && $hello_sha1 blob $hello_size @@ -427,17 +462,23 @@ test_expect_success '--batch-command with multiple info calls gives correct form deadbeef missing EOF - git cat-file --batch-command --buffer >actual <<-EOF && - info $hello_sha1 - info $tree_sha1 - info $commit_sha1 - info $tag_sha1 - info deadbeef - EOF + echo "$batch_command_multiple_info" >in && + git cat-file --batch-command --buffer <in >actual && + + test_cmp expect actual && + + echo "$batch_command_multiple_info" | tr "\n" "\0" >in && + git cat-file --batch-command --buffer -z <in >actual && test_cmp expect actual ' +batch_command_multiple_contents="contents $hello_sha1 +contents $commit_sha1 +contents $tag_sha1 +contents deadbeef +flush" + test_expect_success '--batch-command with multiple command calls gives correct format' ' remove_timestamp >expect <<-EOF && $hello_sha1 blob $hello_size @@ -449,13 +490,14 @@ test_expect_success '--batch-command with multiple command calls gives correct f deadbeef missing EOF - git cat-file --batch-command --buffer >actual_raw <<-EOF && - contents $hello_sha1 - contents $commit_sha1 - contents $tag_sha1 - contents deadbeef - flush - EOF + echo "$batch_command_multiple_contents" >in && + git cat-file --batch-command --buffer <in >actual_raw && + + remove_timestamp <actual_raw >actual && + test_cmp expect actual && + + echo "$batch_command_multiple_contents" | tr "\n" "\0" >in && + git cat-file --batch-command --buffer -z <in >actual_raw && remove_timestamp <actual_raw >actual && test_cmp expect actual diff --git a/t/t1020-subdirectory.sh b/t/t1020-subdirectory.sh index 9fdbb2af80..45eef9457f 100755 --- a/t/t1020-subdirectory.sh +++ b/t/t1020-subdirectory.sh @@ -6,6 +6,7 @@ test_description='Try various core-level commands in subdirectory. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh . "$TEST_DIRECTORY"/lib-read-tree.sh diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 763c6cc684..0302e36fd6 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -372,6 +372,14 @@ test_expect_success 'deep changes during checkout' ' test_all_match git checkout base ' +test_expect_success 'checkout with modified sparse directory' ' + init_repos && + + test_all_match git checkout rename-in-to-out -- . && + test_sparse_match git sparse-checkout reapply && + test_all_match git checkout base +' + test_expect_success 'add outside sparse cone' ' init_repos && @@ -548,7 +556,7 @@ test_expect_success 'blame with pathspec inside sparse definition' ' deep/deeper1/a \ deep/deeper1/deepest/a do - test_all_match git blame $file + test_all_match git blame $file || return 1 done ' @@ -559,7 +567,7 @@ test_expect_success 'blame with pathspec outside sparse definition' ' init_repos && test_sparse_match git sparse-checkout set && - for file in a \ + for file in \ deep/a \ deep/deeper1/a \ deep/deeper1/deepest/a @@ -571,7 +579,7 @@ test_expect_success 'blame with pathspec outside sparse definition' ' # We compare sparse-checkout-err and sparse-index-err in # `test_sparse_match`. Given we know they are the same, we # only check the content of sparse-index-err here. - test_cmp expect sparse-index-err + test_cmp expect sparse-index-err || return 1 done ' @@ -687,6 +695,23 @@ test_expect_success 'reset with wildcard pathspec' ' test_all_match git ls-files -s -- folder1 ' +test_expect_success 'reset hard with removed sparse dir' ' + init_repos && + + run_on_all git rm -r --sparse folder1 && + test_all_match git status --porcelain=v2 && + + test_all_match git reset --hard && + test_all_match git status --porcelain=v2 && + + cat >expect <<-\EOF && + folder1/ + EOF + + git -C sparse-index ls-files --sparse folder1 >out && + test_cmp expect out +' + test_expect_success 'update-index modify outside sparse definition' ' init_repos && @@ -912,7 +937,7 @@ test_expect_success 'read-tree --prefix' ' test_all_match git read-tree --prefix=deep/deeper1/deepest -u deepest && test_all_match git status --porcelain=v2 && - test_all_match git rm -rf --sparse folder1/ && + run_on_all git rm -rf --sparse folder1/ && test_all_match git read-tree --prefix=folder1/ -u update-folder1 && test_all_match git status --porcelain=v2 && @@ -1340,10 +1365,14 @@ ensure_not_expanded () { shift && test_must_fail env \ GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ - git -C sparse-index "$@" || return 1 + git -C sparse-index "$@" \ + >sparse-index-out \ + 2>sparse-index-error || return 1 else GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ - git -C sparse-index "$@" || return 1 + git -C sparse-index "$@" \ + >sparse-index-out \ + 2>sparse-index-error || return 1 fi && test_region ! index ensure_full_index trace2.txt } @@ -1542,7 +1571,7 @@ test_expect_success 'sparse index is not expanded: blame' ' deep/deeper1/a \ deep/deeper1/deepest/a do - ensure_not_expanded blame $file + ensure_not_expanded blame $file || return 1 done ' @@ -1853,4 +1882,94 @@ test_expect_success 'mv directory from out-of-cone to in-cone' ' grep -e "H deep/0/1" actual ' +test_expect_success 'rm pathspec inside sparse definition' ' + init_repos && + + test_all_match git rm deep/a && + test_all_match git status --porcelain=v2 && + + # test wildcard + run_on_all git reset --hard && + test_all_match git rm deep/* && + test_all_match git status --porcelain=v2 && + + # test recursive rm + run_on_all git reset --hard && + test_all_match git rm -r deep && + test_all_match git status --porcelain=v2 +' + +test_expect_success 'rm pathspec outside sparse definition' ' + init_repos && + + for file in folder1/a folder1/0/1 + do + test_sparse_match test_must_fail git rm $file && + test_sparse_match test_must_fail git rm --cached $file && + test_sparse_match git rm --sparse $file && + test_sparse_match git status --porcelain=v2 || return 1 + done && + + cat >folder1-full <<-EOF && + rm ${SQ}folder1/0/0/0${SQ} + rm ${SQ}folder1/0/1${SQ} + rm ${SQ}folder1/a${SQ} + EOF + + cat >folder1-sparse <<-EOF && + rm ${SQ}folder1/${SQ} + EOF + + # test wildcard + run_on_sparse git reset --hard && + run_on_sparse git sparse-checkout reapply && + test_sparse_match test_must_fail git rm folder1/* && + run_on_sparse git rm --sparse folder1/* && + test_cmp folder1-full sparse-checkout-out && + test_cmp folder1-sparse sparse-index-out && + test_sparse_match git status --porcelain=v2 && + + # test recursive rm + run_on_sparse git reset --hard && + run_on_sparse git sparse-checkout reapply && + test_sparse_match test_must_fail git rm --sparse folder1 && + run_on_sparse git rm --sparse -r folder1 && + test_cmp folder1-full sparse-checkout-out && + test_cmp folder1-sparse sparse-index-out && + test_sparse_match git status --porcelain=v2 +' + +test_expect_success 'rm pathspec expands index when necessary' ' + init_repos && + + # in-cone pathspec (do not expand) + ensure_not_expanded rm "deep/deep*" && + test_must_be_empty sparse-index-err && + + # out-of-cone pathspec (expand) + ! ensure_not_expanded rm --sparse "folder1/a*" && + test_must_be_empty sparse-index-err && + + # pathspec that should expand index + ! ensure_not_expanded rm "*/a" && + test_must_be_empty sparse-index-err && + + ! ensure_not_expanded rm "**a" && + test_must_be_empty sparse-index-err +' + +test_expect_success 'sparse index is not expanded: rm' ' + init_repos && + + ensure_not_expanded rm deep/a && + + # test in-cone wildcard + git -C sparse-index reset --hard && + ensure_not_expanded rm deep/* && + + # test recursive rm + git -C sparse-index reset --hard && + ensure_not_expanded rm -r deep +' + test_done diff --git a/t/t1401-symbolic-ref.sh b/t/t1401-symbolic-ref.sh index 9fb0b90f25..0c204089b8 100755 --- a/t/t1401-symbolic-ref.sh +++ b/t/t1401-symbolic-ref.sh @@ -165,4 +165,14 @@ test_expect_success 'symbolic-ref can resolve d/f name (ENOTDIR)' ' test_cmp expect actual ' +test_expect_success 'symbolic-ref refuses invalid target for non-HEAD' ' + test_must_fail git symbolic-ref refs/heads/invalid foo..bar +' + +test_expect_success 'symbolic-ref allows top-level target for non-HEAD' ' + git symbolic-ref refs/heads/top-level FETCH_HEAD && + git update-ref FETCH_HEAD HEAD && + test_cmp_rev top-level HEAD +' + test_done diff --git a/t/t1405-main-ref-store.sh b/t/t1405-main-ref-store.sh index 51f8291628..e4627cf1b6 100755 --- a/t/t1405-main-ref-store.sh +++ b/t/t1405-main-ref-store.sh @@ -5,6 +5,7 @@ test_description='test main ref store api' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh RUN="test-tool ref-store main" diff --git a/t/t1407-worktree-ref-store.sh b/t/t1407-worktree-ref-store.sh index ad8006c813..05b1881c59 100755 --- a/t/t1407-worktree-ref-store.sh +++ b/t/t1407-worktree-ref-store.sh @@ -5,6 +5,7 @@ test_description='test worktree ref store api' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh RWT="test-tool ref-store worktree:wt" diff --git a/t/t1418-reflog-exists.sh b/t/t1418-reflog-exists.sh index d51ecd5e92..2268bca3c1 100755 --- a/t/t1418-reflog-exists.sh +++ b/t/t1418-reflog-exists.sh @@ -4,6 +4,7 @@ test_description='Test reflog display routines' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t1450-fsck.sh b/t/t1450-fsck.sh index ab7f31f1dc..53c2aa10b7 100755 --- a/t/t1450-fsck.sh +++ b/t/t1450-fsck.sh @@ -364,6 +364,20 @@ test_expect_success 'tree entry with type mismatch' ' test_i18ngrep ! "dangling blob" out ' +test_expect_success 'tree entry with bogus mode' ' + test_when_finished "remove_object \$blob" && + test_when_finished "remove_object \$tree" && + blob=$(echo blob | git hash-object -w --stdin) && + blob_oct=$(echo $blob | hex2oct) && + tree=$(printf "100000 foo\0${blob_oct}" | + git hash-object -t tree --stdin -w --literally) && + git fsck 2>err && + cat >expect <<-EOF && + warning in tree $tree: badFilemode: contains bad file modes + EOF + test_cmp expect err +' + test_expect_success 'tag pointing to nonexistent' ' badoid=$(test_oid deadbeef) && cat >invalid-tag <<-EOF && diff --git a/t/t1503-rev-parse-verify.sh b/t/t1503-rev-parse-verify.sh index ba43168d12..bc136833c1 100755 --- a/t/t1503-rev-parse-verify.sh +++ b/t/t1503-rev-parse-verify.sh @@ -132,7 +132,7 @@ test_expect_success 'use --default' ' test_must_fail git rev-parse --verify --default bar ' -test_expect_success !SANITIZE_LEAK 'main@{n} for various n' ' +test_expect_success 'main@{n} for various n' ' git reflog >out && N=$(wc -l <out) && Nm1=$(($N-1)) && diff --git a/t/t1701-racy-split-index.sh b/t/t1701-racy-split-index.sh index 5dc221ef38..d8fa489998 100755 --- a/t/t1701-racy-split-index.sh +++ b/t/t1701-racy-split-index.sh @@ -5,6 +5,7 @@ test_description='racy split index' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t1800-hook.sh b/t/t1800-hook.sh index 210f429887..64096adac7 100755 --- a/t/t1800-hook.sh +++ b/t/t1800-hook.sh @@ -151,4 +151,30 @@ test_expect_success TTY 'git commit: stdout and stderr are connected to a TTY' ' test_hook_tty commit -m"B.new" ' +test_expect_success 'git hook run a hook with a bad shebang' ' + test_when_finished "rm -rf bad-hooks" && + mkdir bad-hooks && + write_script bad-hooks/test-hook "/bad/path/no/spaces" </dev/null && + + # TODO: We should emit the same (or at least a more similar) + # error on Windows and !Windows. See the OS-specific code in + # start_command() + if test_have_prereq !WINDOWS + then + cat >expect <<-\EOF + fatal: cannot run bad-hooks/test-hook: ... + EOF + else + cat >expect <<-\EOF + error: cannot spawn bad-hooks/test-hook: ... + EOF + fi && + test_expect_code 1 git \ + -c core.hooksPath=bad-hooks \ + hook run test-hook >out 2>err && + test_must_be_empty out && + sed -e "s/test-hook: .*/test-hook: .../" <err >actual && + test_cmp expect actual +' + test_done diff --git a/t/t2006-checkout-index-basic.sh b/t/t2006-checkout-index-basic.sh index 7705e3a317..5d119871d4 100755 --- a/t/t2006-checkout-index-basic.sh +++ b/t/t2006-checkout-index-basic.sh @@ -3,6 +3,7 @@ test_description='basic checkout-index tests ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'checkout-index --gobbledegook' ' diff --git a/t/t2020-checkout-detach.sh b/t/t2020-checkout-detach.sh index bc46713a43..2eab6474f8 100755 --- a/t/t2020-checkout-detach.sh +++ b/t/t2020-checkout-detach.sh @@ -4,6 +4,7 @@ test_description='checkout into detached HEAD state' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh check_detached () { diff --git a/t/t2023-checkout-m.sh b/t/t2023-checkout-m.sh index 7b327b7544..81e772fb4e 100755 --- a/t/t2023-checkout-m.sh +++ b/t/t2023-checkout-m.sh @@ -7,6 +7,7 @@ Ensures that checkout -m on a resolved file restores the conflicted file' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t2080-parallel-checkout-basics.sh b/t/t2080-parallel-checkout-basics.sh index 3e0f8c675f..00ce3033d3 100755 --- a/t/t2080-parallel-checkout-basics.sh +++ b/t/t2080-parallel-checkout-basics.sh @@ -226,4 +226,49 @@ test_expect_success SYMLINKS 'parallel checkout checks for symlinks in leading d ) ' +# This test is here (and not in e.g. t2022-checkout-paths.sh), because we +# check the final report including sequential, parallel, and delayed entries +# all at the same time. So we must have finer control of the parallel checkout +# variables. +test_expect_success '"git checkout ." report should not include failed entries' ' + test_config_global filter.delay.process \ + "test-tool rot13-filter --always-delay --log=delayed.log clean smudge delay" && + test_config_global filter.delay.required true && + test_config_global filter.cat.clean cat && + test_config_global filter.cat.smudge cat && + test_config_global filter.cat.required true && + + set_checkout_config 2 0 && + git init failed_entries && + ( + cd failed_entries && + cat >.gitattributes <<-EOF && + *delay* filter=delay + parallel-ineligible* filter=cat + EOF + echo a >missing-delay.a && + echo a >parallel-ineligible.a && + echo a >parallel-eligible.a && + echo b >success-delay.b && + echo b >parallel-ineligible.b && + echo b >parallel-eligible.b && + git add -A && + git commit -m files && + + a_blob="$(git rev-parse :parallel-ineligible.a)" && + rm .git/objects/$(test_oid_to_path $a_blob) && + rm *.a *.b && + + test_checkout_workers 2 test_must_fail git checkout . 2>err && + + # All *.b entries should succeed and all *.a entries should fail: + # - missing-delay.a: the delay filter will drop this path + # - parallel-*.a: the blob will be missing + # + grep "Updated 3 paths from the index" err && + test_stdout_line_count = 3 ls *.b && + ! ls *.a + ) +' + test_done diff --git a/t/t2082-parallel-checkout-attributes.sh b/t/t2082-parallel-checkout-attributes.sh index 2525457961..f3511cd43a 100755 --- a/t/t2082-parallel-checkout-attributes.sh +++ b/t/t2082-parallel-checkout-attributes.sh @@ -138,12 +138,9 @@ test_expect_success 'parallel-checkout and external filter' ' # The delayed queue is independent from the parallel queue, and they should be # able to work together in the same checkout process. # -test_expect_success PERL 'parallel-checkout and delayed checkout' ' - write_script rot13-filter.pl "$PERL_PATH" \ - <"$TEST_DIRECTORY"/t0021/rot13-filter.pl && - +test_expect_success 'parallel-checkout and delayed checkout' ' test_config_global filter.delay.process \ - "\"$(pwd)/rot13-filter.pl\" --always-delay \"$(pwd)/delayed.log\" clean smudge delay" && + "test-tool rot13-filter --always-delay --log=\"$(pwd)/delayed.log\" clean smudge delay" && test_config_global filter.delay.required true && echo "abcd" >original && diff --git a/t/t2205-add-worktree-config.sh b/t/t2205-add-worktree-config.sh index 43d950de64..98265ba1b4 100755 --- a/t/t2205-add-worktree-config.sh +++ b/t/t2205-add-worktree-config.sh @@ -17,6 +17,7 @@ outside the repository. Two instances for which this can occur are tested: repository can be added to the index. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success '1a: setup--config worktree' ' diff --git a/t/t2407-worktree-heads.sh b/t/t2407-worktree-heads.sh index b6be42f74a..019a40df2c 100755 --- a/t/t2407-worktree-heads.sh +++ b/t/t2407-worktree-heads.sh @@ -7,13 +7,18 @@ TEST_PASSES_SANITIZE_LEAK=true test_expect_success 'setup' ' test_commit init && - git branch -f fake-1 && - git branch -f fake-2 && for i in 1 2 3 4 do + git checkout -b conflict-$i && + echo "not I" >$i.t && + git add $i.t && + git commit -m "will conflict" && + + git checkout - && test_commit $i && git branch wt-$i && + git branch fake-$i && git worktree add wt-$i wt-$i || return 1 done && @@ -36,34 +41,58 @@ test_expect_success 'setup' ' test_expect_success 'refuse to overwrite: checked out in worktree' ' for i in 1 2 3 4 do - test_must_fail git branch -f wt-$i HEAD 2>err + test_must_fail git branch -f wt-$i HEAD 2>err && grep "cannot force update the branch" err && - test_must_fail git branch -D wt-$i 2>err + test_must_fail git branch -D wt-$i 2>err && grep "Cannot delete branch" err || return 1 done ' -test_expect_success 'refuse to overwrite: worktree in bisect' ' - test_when_finished rm -rf .git/worktrees/wt-*/BISECT_* && +test_expect_success !SANITIZE_LEAK 'refuse to overwrite: worktree in bisect' ' + test_when_finished git -C wt-4 bisect reset && - touch .git/worktrees/wt-4/BISECT_LOG && - echo refs/heads/fake-2 >.git/worktrees/wt-4/BISECT_START && + # Set up a bisect so HEAD no longer points to wt-4. + git -C wt-4 bisect start && + git -C wt-4 bisect bad wt-4 && + git -C wt-4 bisect good wt-1 && - test_must_fail git branch -f fake-2 HEAD 2>err && - grep "cannot force update the branch '\''fake-2'\'' checked out at.*wt-4" err + test_must_fail git branch -f wt-4 HEAD 2>err && + grep "cannot force update the branch '\''wt-4'\'' checked out at.*wt-4" err ' -test_expect_success 'refuse to overwrite: worktree in rebase' ' - test_when_finished rm -rf .git/worktrees/wt-*/rebase-merge && +test_expect_success !SANITIZE_LEAK 'refuse to overwrite: worktree in rebase (apply)' ' + test_when_finished git -C wt-2 rebase --abort && - mkdir -p .git/worktrees/wt-3/rebase-merge && - touch .git/worktrees/wt-3/rebase-merge/interactive && - echo refs/heads/fake-1 >.git/worktrees/wt-3/rebase-merge/head-name && - echo refs/heads/fake-2 >.git/worktrees/wt-3/rebase-merge/onto && + # This will fail part-way through due to a conflict. + test_must_fail git -C wt-2 rebase --apply conflict-2 && + + test_must_fail git branch -f wt-2 HEAD 2>err && + grep "cannot force update the branch '\''wt-2'\'' checked out at.*wt-2" err +' + +test_expect_success !SANITIZE_LEAK 'refuse to overwrite: worktree in rebase (merge)' ' + test_when_finished git -C wt-2 rebase --abort && - test_must_fail git branch -f fake-1 HEAD 2>err && - grep "cannot force update the branch '\''fake-1'\'' checked out at.*wt-3" err + # This will fail part-way through due to a conflict. + test_must_fail git -C wt-2 rebase conflict-2 && + + test_must_fail git branch -f wt-2 HEAD 2>err && + grep "cannot force update the branch '\''wt-2'\'' checked out at.*wt-2" err +' + +test_expect_success !SANITIZE_LEAK 'refuse to overwrite: worktree in rebase with --update-refs' ' + test_when_finished git -C wt-3 rebase --abort && + + git branch -f can-be-updated wt-3 && + test_must_fail git -C wt-3 rebase --update-refs conflict-3 && + + for i in 3 4 + do + test_must_fail git branch -f can-be-updated HEAD 2>err && + grep "cannot force update the branch '\''can-be-updated'\'' checked out at.*wt-3" err || + return 1 + done ' test_expect_success !SANITIZE_LEAK 'refuse to fetch over ref: checked out' ' @@ -77,24 +106,24 @@ test_expect_success !SANITIZE_LEAK 'refuse to fetch over ref: checked out' ' ' test_expect_success !SANITIZE_LEAK 'refuse to fetch over ref: worktree in bisect' ' - test_when_finished rm -rf .git/worktrees/wt-*/BISECT_* && + test_when_finished git -C wt-4 bisect reset && - touch .git/worktrees/wt-4/BISECT_LOG && - echo refs/heads/fake-2 >.git/worktrees/wt-4/BISECT_START && + # Set up a bisect so HEAD no longer points to wt-4. + git -C wt-4 bisect start && + git -C wt-4 bisect bad wt-4 && + git -C wt-4 bisect good wt-1 && - test_must_fail git fetch server +refs/heads/fake-2:refs/heads/fake-2 2>err && + test_must_fail git fetch server +refs/heads/wt-4:refs/heads/wt-4 2>err && grep "refusing to fetch into branch" err ' test_expect_success !SANITIZE_LEAK 'refuse to fetch over ref: worktree in rebase' ' - test_when_finished rm -rf .git/worktrees/wt-*/rebase-merge && + test_when_finished git -C wt-3 rebase --abort && - mkdir -p .git/worktrees/wt-4/rebase-merge && - touch .git/worktrees/wt-4/rebase-merge/interactive && - echo refs/heads/fake-1 >.git/worktrees/wt-4/rebase-merge/head-name && - echo refs/heads/fake-2 >.git/worktrees/wt-4/rebase-merge/onto && + # This will fail part-way through due to a conflict. + test_must_fail git -C wt-3 rebase conflict-3 && - test_must_fail git fetch server +refs/heads/fake-1:refs/heads/fake-1 2>err && + test_must_fail git fetch server +refs/heads/wt-3:refs/heads/wt-3 2>err && grep "refusing to fetch into branch" err ' @@ -126,4 +155,26 @@ test_expect_success 'refuse to overwrite when in error states' ' done ' +. "$TEST_DIRECTORY"/lib-rebase.sh + +test_expect_success !SANITIZE_LEAK 'refuse to overwrite during rebase with --update-refs' ' + git commit --fixup HEAD~2 --allow-empty && + ( + set_cat_todo_editor && + test_must_fail git rebase -i --update-refs HEAD~3 >todo && + ! grep "update-refs" todo + ) && + git branch -f allow-update HEAD~2 && + ( + set_cat_todo_editor && + test_must_fail git rebase -i --update-refs HEAD~3 >todo && + grep "update-ref refs/heads/allow-update" todo + ) +' + +# This must be the last test in this file +test_expect_success '$EDITOR and friends are unchanged' ' + test_editor_unchanged +' + test_done diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh index e07ac6c6dc..1ed0aa967e 100755 --- a/t/t3001-ls-files-others-exclude.sh +++ b/t/t3001-ls-files-others-exclude.sh @@ -103,7 +103,7 @@ test_expect_success 'git ls-files --others with various exclude options.' ' test_cmp expect output ' -test_expect_success !SANITIZE_LEAK 'restore gitignore' ' +test_expect_success 'restore gitignore' ' git checkout --ignore-skip-worktree-bits $allignores && rm .git/index ' @@ -126,7 +126,7 @@ cat > expect << EOF # three/ EOF -test_expect_success !SANITIZE_LEAK 'git status honors core.excludesfile' \ +test_expect_success 'git status honors core.excludesfile' \ 'test_cmp expect output' test_expect_success 'trailing slash in exclude allows directory match(1)' ' diff --git a/t/t3012-ls-files-dedup.sh b/t/t3012-ls-files-dedup.sh index 2682b1f43a..190e2f6eed 100755 --- a/t/t3012-ls-files-dedup.sh +++ b/t/t3012-ls-files-dedup.sh @@ -2,6 +2,7 @@ test_description='git ls-files --deduplicate test' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3013-ls-files-format.sh b/t/t3013-ls-files-format.sh new file mode 100755 index 0000000000..efb7450bf1 --- /dev/null +++ b/t/t3013-ls-files-format.sh @@ -0,0 +1,95 @@ +#!/bin/sh + +test_description='git ls-files --format test' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +for flag in -s -o -k -t --resolve-undo --deduplicate --eol +do + test_expect_success "usage: --format is incompatible with $flag" ' + test_expect_code 129 git ls-files --format="%(objectname)" $flag + ' +done + +test_expect_success 'setup' ' + printf "LINEONE\nLINETWO\nLINETHREE\n" >o1.txt && + printf "LINEONE\r\nLINETWO\r\nLINETHREE\r\n" >o2.txt && + printf "LINEONE\r\nLINETWO\nLINETHREE\n" >o3.txt && + git add o?.txt && + oid=$(git hash-object o1.txt) && + git update-index --add --cacheinfo 120000 $oid o4.txt && + git update-index --add --cacheinfo 160000 $oid o5.txt && + git update-index --add --cacheinfo 100755 $oid o6.txt && + git commit -m base +' + +test_expect_success 'git ls-files --format objectmode v.s. -s' ' + git ls-files -s >files && + cut -d" " -f1 files >expect && + git ls-files --format="%(objectmode)" >actual && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format objectname v.s. -s' ' + git ls-files -s >files && + cut -d" " -f2 files >expect && + git ls-files --format="%(objectname)" >actual && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format v.s. --eol' ' + git ls-files --eol >tmp && + sed -e "s/ / /g" -e "s/ */ /g" tmp >expect 2>err && + test_must_be_empty err && + git ls-files --format="i/%(eolinfo:index) w/%(eolinfo:worktree) attr/%(eolattr) %(path)" >actual 2>err && + test_must_be_empty err && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format path v.s. -s' ' + git ls-files -s >files && + cut -f2 files >expect && + git ls-files --format="%(path)" >actual && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format with -m' ' + echo change >o1.txt && + cat >expect <<-\EOF && + o1.txt + o4.txt + o5.txt + o6.txt + EOF + git ls-files --format="%(path)" -m >actual && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format with -d' ' + echo o7 >o7.txt && + git add o7.txt && + rm o7.txt && + cat >expect <<-\EOF && + o4.txt + o5.txt + o6.txt + o7.txt + EOF + git ls-files --format="%(path)" -d >actual && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format v.s -s' ' + git ls-files --stage >expect && + git ls-files --format="%(objectmode) %(objectname) %(stage)%x09%(path)" >actual && + test_cmp expect actual +' + +test_expect_success 'git ls-files --format with --debug' ' + git ls-files --debug >expect && + git ls-files --format="%(path)" --debug >actual && + test_cmp expect actual +' + +test_done diff --git a/t/t3305-notes-fanout.sh b/t/t3305-notes-fanout.sh index 64a9915761..22ffe5bcb9 100755 --- a/t/t3305-notes-fanout.sh +++ b/t/t3305-notes-fanout.sh @@ -51,7 +51,7 @@ test_expect_success 'creating many notes with git-notes' ' done ' -test_expect_success !SANITIZE_LEAK 'many notes created correctly with git-notes' ' +test_expect_success 'many notes created correctly with git-notes' ' git log >output.raw && grep "^ " output.raw >output && i=$num_notes && diff --git a/t/t3307-notes-man.sh b/t/t3307-notes-man.sh index 1aa366a410..ae316502c4 100755 --- a/t/t3307-notes-man.sh +++ b/t/t3307-notes-man.sh @@ -4,6 +4,7 @@ test_description='Examples from the git-notes man page Make sure the manual is not full of lies.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index f31afd4a54..688b01e3eb 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -1743,6 +1743,279 @@ test_expect_success 'ORIG_HEAD is updated correctly' ' test_cmp_rev ORIG_HEAD test-orig-head@{1} ' +test_expect_success '--update-refs adds label and update-ref commands' ' + git checkout -b update-refs no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + git commit --allow-empty --fixup=third && + git branch -f is-not-reordered && + git commit --allow-empty --fixup=HEAD~4 && + git branch -f shared-tip && + ( + set_cat_todo_editor && + + cat >expect <<-EOF && + pick $(git log -1 --format=%h J) J + fixup $(git log -1 --format=%h update-refs) fixup! J # empty + update-ref refs/heads/second + update-ref refs/heads/first + pick $(git log -1 --format=%h K) K + pick $(git log -1 --format=%h L) L + fixup $(git log -1 --format=%h is-not-reordered) fixup! L # empty + update-ref refs/heads/third + pick $(git log -1 --format=%h M) M + update-ref refs/heads/no-conflict-branch + update-ref refs/heads/is-not-reordered + update-ref refs/heads/shared-tip + EOF + + test_must_fail git rebase -i --autosquash --update-refs primary >todo && + test_cmp expect todo && + + test_must_fail git -c rebase.autosquash=true \ + -c rebase.updaterefs=true \ + rebase -i primary >todo && + + test_cmp expect todo + ) +' + +test_expect_success '--update-refs adds commands with --rebase-merges' ' + git checkout -b update-refs-with-merge no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + git merge -m merge branch2 && + git branch -f merge-branch && + git commit --fixup=third --allow-empty && + ( + set_cat_todo_editor && + + cat >expect <<-EOF && + label onto + reset onto + pick $(git log -1 --format=%h branch2~1) F + pick $(git log -1 --format=%h branch2) I + update-ref refs/heads/branch2 + label merge + reset onto + pick $(git log -1 --format=%h refs/heads/second) J + update-ref refs/heads/second + update-ref refs/heads/first + pick $(git log -1 --format=%h refs/heads/third~1) K + pick $(git log -1 --format=%h refs/heads/third) L + fixup $(git log -1 --format=%h update-refs-with-merge) fixup! L # empty + update-ref refs/heads/third + pick $(git log -1 --format=%h HEAD~2) M + update-ref refs/heads/no-conflict-branch + merge -C $(git log -1 --format=%h HEAD~1) merge # merge + update-ref refs/heads/merge-branch + EOF + + test_must_fail git rebase -i --autosquash \ + --rebase-merges=rebase-cousins \ + --update-refs primary >todo && + + test_cmp expect todo && + + test_must_fail git -c rebase.autosquash=true \ + -c rebase.updaterefs=true \ + rebase -i \ + --rebase-merges=rebase-cousins \ + primary >todo && + + test_cmp expect todo + ) +' + +test_expect_success '--update-refs updates refs correctly' ' + git checkout -B update-refs no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + test_commit extra2 fileX && + git commit --amend --fixup=L && + + git rebase -i --autosquash --update-refs primary 2>err && + + test_cmp_rev HEAD~3 refs/heads/first && + test_cmp_rev HEAD~3 refs/heads/second && + test_cmp_rev HEAD~1 refs/heads/third && + test_cmp_rev HEAD refs/heads/no-conflict-branch && + + cat >expect <<-\EOF && + Successfully rebased and updated refs/heads/update-refs. + Updated the following refs with --update-refs: + refs/heads/first + refs/heads/no-conflict-branch + refs/heads/second + refs/heads/third + EOF + + # Clear "Rebasing (X/Y)" progress lines and drop leading tabs. + sed -e "s/Rebasing.*Successfully/Successfully/g" -e "s/^\t//g" \ + <err >err.trimmed && + test_cmp expect err.trimmed +' + +test_expect_success 'respect user edits to update-ref steps' ' + git checkout -B update-refs-break no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~3 && + git branch -f third HEAD~1 && + git branch -f unseen base && + + # First, we will add breaks to the expected todo file + cat >fake-todo-1 <<-EOF && + pick $(git rev-parse HEAD~3) + break + update-ref refs/heads/second + update-ref refs/heads/first + + pick $(git rev-parse HEAD~2) + pick $(git rev-parse HEAD~1) + update-ref refs/heads/third + + pick $(git rev-parse HEAD) + update-ref refs/heads/no-conflict-branch + EOF + + # Second, we will drop some update-refs commands (and move one) + cat >fake-todo-2 <<-EOF && + update-ref refs/heads/second + + pick $(git rev-parse HEAD~2) + update-ref refs/heads/third + pick $(git rev-parse HEAD~1) + break + + pick $(git rev-parse HEAD) + EOF + + # Third, we will: + # * insert a new one (new-branch), + # * re-add an old one (first), and + # * add a second instance of a previously-stored one (second) + cat >fake-todo-3 <<-EOF && + update-ref refs/heads/unseen + update-ref refs/heads/new-branch + pick $(git rev-parse HEAD) + update-ref refs/heads/first + update-ref refs/heads/second + EOF + + ( + set_replace_editor fake-todo-1 && + git rebase -i --update-refs primary && + + # These branches are currently locked. + for b in first second third no-conflict-branch + do + test_must_fail git branch -f $b base || return 1 + done && + + set_replace_editor fake-todo-2 && + git rebase --edit-todo && + + # These branches are currently locked. + for b in second third + do + test_must_fail git branch -f $b base || return 1 + done && + + # These branches are currently unlocked for checkout. + for b in first no-conflict-branch + do + git worktree add wt-$b $b && + git worktree remove wt-$b || return 1 + done && + + git rebase --continue && + + set_replace_editor fake-todo-3 && + git rebase --edit-todo && + + # These branches are currently locked. + for b in second third first unseen + do + test_must_fail git branch -f $b base || return 1 + done && + + # These branches are currently unlocked for checkout. + for b in no-conflict-branch + do + git worktree add wt-$b $b && + git worktree remove wt-$b || return 1 + done && + + git rebase --continue + ) && + + test_cmp_rev HEAD~2 refs/heads/third && + test_cmp_rev HEAD~1 refs/heads/unseen && + test_cmp_rev HEAD~1 refs/heads/new-branch && + test_cmp_rev HEAD refs/heads/first && + test_cmp_rev HEAD refs/heads/second && + test_cmp_rev HEAD refs/heads/no-conflict-branch +' + +test_expect_success '--update-refs: check failed ref update' ' + git checkout -B update-refs-error no-conflict-branch && + git branch -f base HEAD~4 && + git branch -f first HEAD~3 && + git branch -f second HEAD~2 && + git branch -f third HEAD~1 && + + cat >fake-todo <<-EOF && + pick $(git rev-parse HEAD~3) + break + update-ref refs/heads/first + + pick $(git rev-parse HEAD~2) + update-ref refs/heads/second + + pick $(git rev-parse HEAD~1) + update-ref refs/heads/third + + pick $(git rev-parse HEAD) + update-ref refs/heads/no-conflict-branch + EOF + + ( + set_replace_editor fake-todo && + git rebase -i --update-refs base + ) && + + # At this point, the values of first, second, and third are + # recorded in the update-refs file. We will force-update the + # "second" ref, but "git branch -f" will not work because of + # the lock in the update-refs file. + git rev-parse third >.git/refs/heads/second && + + test_must_fail git rebase --continue 2>err && + grep "update_ref failed for ref '\''refs/heads/second'\''" err && + + cat >expect <<-\EOF && + Updated the following refs with --update-refs: + refs/heads/first + refs/heads/no-conflict-branch + refs/heads/third + Failed to update the following refs with --update-refs: + refs/heads/second + EOF + + # Clear "Rebasing (X/Y)" progress lines and drop leading tabs. + tail -n 6 err >err.last && + sed -e "s/Rebasing.*Successfully/Successfully/g" -e "s/^\t//g" \ + <err.last >err.trimmed && + test_cmp expect err.trimmed +' + # This must be the last test in this file test_expect_success '$EDITOR and friends are unchanged' ' test_editor_unchanged diff --git a/t/t3701-add-interactive.sh b/t/t3701-add-interactive.sh index b354fb39de..3b7df9bed5 100755 --- a/t/t3701-add-interactive.sh +++ b/t/t3701-add-interactive.sh @@ -766,6 +766,19 @@ test_expect_success 'detect bogus diffFilter output' ' force_color test_must_fail git add -p <y ' +test_expect_success 'handle very large filtered diff' ' + git reset --hard && + # The specific number here is not important, but it must + # be large enough that the output of "git diff --color" + # fills up the pipe buffer. 10,000 results in ~200k of + # colored output. + test_seq 10000 >test && + test_config interactive.diffFilter cat && + printf y >y && + force_color git add -p >output 2>&1 <y && + git diff-files --exit-code -- test +' + test_expect_success 'diff.algorithm is passed to `git diff-files`' ' git reset --hard && diff --git a/t/t3920-crlf-messages.sh b/t/t3920-crlf-messages.sh index 0276edbe3d..4c661d4d54 100755 --- a/t/t3920-crlf-messages.sh +++ b/t/t3920-crlf-messages.sh @@ -1,6 +1,8 @@ #!/bin/sh test_description='Test ref-filter and pretty APIs for commit and tag messages using CRLF' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh LIB_CRLF_BRANCHES="" diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index 056e922164..dfcf3a0aaa 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -352,6 +352,8 @@ log -GF -p --pickaxe-all master log -IA -IB -I1 -I2 -p master log --decorate --all log --decorate=full --all +log --decorate --clear-decorations --all +log --decorate=full --clear-decorations --all rev-list --parents HEAD rev-list --children HEAD diff --git a/t/t4013/diff.log_--decorate=full_--all b/t/t4013/diff.log_--decorate=full_--all index 3f9b872ece..6b0b334a5d 100644 --- a/t/t4013/diff.log_--decorate=full_--all +++ b/t/t4013/diff.log_--decorate=full_--all @@ -20,7 +20,7 @@ Date: Mon Jun 26 00:06:00 2006 +0000 Rearranged lines in dir/sub -commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 (refs/notes/commits) +commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 Author: A U Thor <author@example.com> Date: Mon Jun 26 00:06:00 2006 +0000 diff --git a/t/t4013/diff.log_--decorate=full_--clear-decorations_--all b/t/t4013/diff.log_--decorate=full_--clear-decorations_--all new file mode 100644 index 0000000000..1c030a6554 --- /dev/null +++ b/t/t4013/diff.log_--decorate=full_--clear-decorations_--all @@ -0,0 +1,61 @@ +$ git log --decorate=full --clear-decorations --all +commit b7e0bc69303b488b47deca799a7d723971dfa6cd (refs/heads/mode) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode + +commit a6f364368ca320bc5a92e18912e16fa6b3dff598 (refs/heads/note) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode (file2) + +Notes: + note + +commit cd4e72fd96faed3f0ba949dc42967430374e2290 (refs/heads/rearrange) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Rearranged lines in dir/sub + +commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 (refs/notes/commits) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Notes added by 'git notes add' + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (HEAD -> refs/heads/master) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (refs/heads/side) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + +commit 444ac553ac7612cc88969031b02b3767fb8a353a (refs/heads/initial) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ diff --git a/t/t4013/diff.log_--decorate=full_--decorate-all_--all b/t/t4013/diff.log_--decorate=full_--decorate-all_--all new file mode 100644 index 0000000000..d6e7928784 --- /dev/null +++ b/t/t4013/diff.log_--decorate=full_--decorate-all_--all @@ -0,0 +1,61 @@ +$ git log --decorate=full --decorate-all --all +commit b7e0bc69303b488b47deca799a7d723971dfa6cd (refs/heads/mode) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode + +commit a6f364368ca320bc5a92e18912e16fa6b3dff598 (refs/heads/note) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode (file2) + +Notes: + note + +commit cd4e72fd96faed3f0ba949dc42967430374e2290 (refs/heads/rearrange) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Rearranged lines in dir/sub + +commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 (refs/notes/commits) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Notes added by 'git notes add' + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (HEAD -> refs/heads/master) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (refs/heads/side) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + +commit 444ac553ac7612cc88969031b02b3767fb8a353a (refs/heads/initial) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ diff --git a/t/t4013/diff.log_--decorate_--all b/t/t4013/diff.log_--decorate_--all index f5e20e1e14..c7df1f5814 100644 --- a/t/t4013/diff.log_--decorate_--all +++ b/t/t4013/diff.log_--decorate_--all @@ -20,7 +20,7 @@ Date: Mon Jun 26 00:06:00 2006 +0000 Rearranged lines in dir/sub -commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 (refs/notes/commits) +commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 Author: A U Thor <author@example.com> Date: Mon Jun 26 00:06:00 2006 +0000 diff --git a/t/t4013/diff.log_--decorate_--clear-decorations_--all b/t/t4013/diff.log_--decorate_--clear-decorations_--all new file mode 100644 index 0000000000..88be82cce3 --- /dev/null +++ b/t/t4013/diff.log_--decorate_--clear-decorations_--all @@ -0,0 +1,61 @@ +$ git log --decorate --clear-decorations --all +commit b7e0bc69303b488b47deca799a7d723971dfa6cd (mode) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode + +commit a6f364368ca320bc5a92e18912e16fa6b3dff598 (note) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode (file2) + +Notes: + note + +commit cd4e72fd96faed3f0ba949dc42967430374e2290 (rearrange) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Rearranged lines in dir/sub + +commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 (refs/notes/commits) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Notes added by 'git notes add' + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (HEAD -> master) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (side) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + +commit 444ac553ac7612cc88969031b02b3767fb8a353a (initial) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ diff --git a/t/t4013/diff.log_--decorate_--decorate-all_--all b/t/t4013/diff.log_--decorate_--decorate-all_--all new file mode 100644 index 0000000000..5d22618bb6 --- /dev/null +++ b/t/t4013/diff.log_--decorate_--decorate-all_--all @@ -0,0 +1,61 @@ +$ git log --decorate --decorate-all --all +commit b7e0bc69303b488b47deca799a7d723971dfa6cd (mode) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode + +commit a6f364368ca320bc5a92e18912e16fa6b3dff598 (note) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + update mode (file2) + +Notes: + note + +commit cd4e72fd96faed3f0ba949dc42967430374e2290 (rearrange) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Rearranged lines in dir/sub + +commit cbacedd14cb8b89255a2c02b59e77a2e9a8021a0 (refs/notes/commits) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:06:00 2006 +0000 + + Notes added by 'git notes add' + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (HEAD -> master) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (side) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + +commit 444ac553ac7612cc88969031b02b3767fb8a353a (initial) +Author: A U Thor <author@example.com> +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ diff --git a/t/t4017-diff-retval.sh b/t/t4017-diff-retval.sh index ed461f481e..5bc28ad9f0 100755 --- a/t/t4017-diff-retval.sh +++ b/t/t4017-diff-retval.sh @@ -5,6 +5,7 @@ test_description='Return value of diffs' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t4020-diff-external.sh b/t/t4020-diff-external.sh index 858a5522f9..c1ac09ecc7 100755 --- a/t/t4020-diff-external.sh +++ b/t/t4020-diff-external.sh @@ -33,7 +33,7 @@ test_expect_success 'GIT_EXTERNAL_DIFF environment' ' ' -test_expect_success !SANITIZE_LEAK 'GIT_EXTERNAL_DIFF environment should apply only to diff' ' +test_expect_success 'GIT_EXTERNAL_DIFF environment should apply only to diff' ' GIT_EXTERNAL_DIFF=echo git log -p -1 HEAD >out && grep "^diff --git a/file b/file" out @@ -74,7 +74,7 @@ test_expect_success 'diff.external' ' test_cmp expect actual ' -test_expect_success !SANITIZE_LEAK 'diff.external should apply only to diff' ' +test_expect_success 'diff.external should apply only to diff' ' test_config diff.external echo && git log -p -1 HEAD >out && grep "^diff --git a/file b/file" out diff --git a/t/t4051-diff-function-context.sh b/t/t4051-diff-function-context.sh index 4838a1df8b..725278ad19 100755 --- a/t/t4051-diff-function-context.sh +++ b/t/t4051-diff-function-context.sh @@ -2,6 +2,7 @@ test_description='diff function context' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh dir="$TEST_DIRECTORY/t4051" diff --git a/t/t4057-diff-combined-paths.sh b/t/t4057-diff-combined-paths.sh index 04b8a1542a..9a7505cbb8 100755 --- a/t/t4057-diff-combined-paths.sh +++ b/t/t4057-diff-combined-paths.sh @@ -5,6 +5,7 @@ test_description='combined diff show only paths that are different to all parent GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # verify that diffc.expect matches output of diff --git a/t/t4069-remerge-diff.sh b/t/t4069-remerge-diff.sh index 35f94957fc..9e7cac68b1 100755 --- a/t/t4069-remerge-diff.sh +++ b/t/t4069-remerge-diff.sh @@ -2,6 +2,7 @@ test_description='remerge-diff handling' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # This test is ort-specific diff --git a/t/t4114-apply-typechange.sh b/t/t4114-apply-typechange.sh index da3e64f811..8ff3640766 100755 --- a/t/t4114-apply-typechange.sh +++ b/t/t4114-apply-typechange.sh @@ -7,6 +7,7 @@ test_description='git apply should not get confused with type changes. ' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup repository and commits' ' diff --git a/t/t4200-rerere.sh b/t/t4200-rerere.sh index 9f8c76dffb..7025cfdae5 100755 --- a/t/t4200-rerere.sh +++ b/t/t4200-rerere.sh @@ -365,9 +365,6 @@ test_expect_success 'set up an unresolved merge' ' test_might_fail git config --unset rerere.autoupdate && git reset --hard && git checkout version2 && - fifth=$(git rev-parse fifth) && - echo "$fifth branch fifth of ." | - git fmt-merge-msg >msg && ancestor=$(git merge-base version2 fifth) && test_must_fail git merge-recursive "$ancestor" -- HEAD fifth && diff --git a/t/t4202-log.sh b/t/t4202-log.sh index 6e66352558..cc15cb4ff6 100755 --- a/t/t4202-log.sh +++ b/t/t4202-log.sh @@ -704,9 +704,12 @@ test_expect_success 'set up more tangled history' ' git checkout -b tangle HEAD~6 && test_commit tangle-a tangle-a a && git merge main~3 && + git update-ref refs/prefetch/merge HEAD && git merge side~1 && + git update-ref refs/rewritten/merge HEAD && git checkout main && git merge tangle && + git update-ref refs/hidden/tangle HEAD && git checkout -b reach && test_commit reach && git checkout main && @@ -974,9 +977,9 @@ test_expect_success 'decorate-refs-exclude and simplify-by-decoration' ' Merge-tag-reach (HEAD -> main) reach (tag: reach, reach) seventh (tag: seventh) - Merge-branch-tangle - Merge-branch-side-early-part-into-tangle (tangle) - tangle-a (tag: tangle-a) + Merge-branch-tangle (refs/hidden/tangle) + Merge-branch-side-early-part-into-tangle (refs/rewritten/merge, tangle) + Merge-branch-main-early-part-into-tangle (refs/prefetch/merge) EOF git log -n6 --decorate=short --pretty="tformat:%f%d" \ --decorate-refs-exclude="*octopus*" \ @@ -1025,6 +1028,115 @@ test_expect_success 'decorate-refs and simplify-by-decoration without output' ' test_cmp expect actual ' +test_expect_success 'decorate-refs-exclude HEAD' ' + git log --decorate=full --oneline \ + --decorate-refs-exclude="HEAD" >actual && + ! grep HEAD actual +' + +test_expect_success 'decorate-refs focus from default' ' + git log --decorate=full --oneline \ + --decorate-refs="refs/heads" >actual && + ! grep HEAD actual +' + +test_expect_success '--clear-decorations overrides defaults' ' + cat >expect.default <<-\EOF && + Merge-tag-reach (HEAD -> refs/heads/main) + Merge-tags-octopus-a-and-octopus-b + seventh (tag: refs/tags/seventh) + octopus-b (tag: refs/tags/octopus-b, refs/heads/octopus-b) + octopus-a (tag: refs/tags/octopus-a, refs/heads/octopus-a) + reach (tag: refs/tags/reach, refs/heads/reach) + Merge-branch-tangle + Merge-branch-side-early-part-into-tangle (refs/heads/tangle) + Merge-branch-main-early-part-into-tangle + tangle-a (tag: refs/tags/tangle-a) + Merge-branch-side + side-2 (tag: refs/tags/side-2, refs/heads/side) + side-1 (tag: refs/tags/side-1) + Second + sixth + fifth + fourth + third + second + initial + EOF + git log --decorate=full --pretty="tformat:%f%d" >actual && + test_cmp expect.default actual && + + cat >expect.all <<-\EOF && + Merge-tag-reach (HEAD -> refs/heads/main) + Merge-tags-octopus-a-and-octopus-b + seventh (tag: refs/tags/seventh) + octopus-b (tag: refs/tags/octopus-b, refs/heads/octopus-b) + octopus-a (tag: refs/tags/octopus-a, refs/heads/octopus-a) + reach (tag: refs/tags/reach, refs/heads/reach) + Merge-branch-tangle (refs/hidden/tangle) + Merge-branch-side-early-part-into-tangle (refs/rewritten/merge, refs/heads/tangle) + Merge-branch-main-early-part-into-tangle (refs/prefetch/merge) + tangle-a (tag: refs/tags/tangle-a) + Merge-branch-side + side-2 (tag: refs/tags/side-2, refs/heads/side) + side-1 (tag: refs/tags/side-1) + Second + sixth + fifth + fourth + third + second + initial + EOF + git log --decorate=full --pretty="tformat:%f%d" \ + --clear-decorations >actual && + test_cmp expect.all actual && + git -c log.initialDecorationSet=all log \ + --decorate=full --pretty="tformat:%f%d" >actual && + test_cmp expect.all actual +' + +test_expect_success '--clear-decorations clears previous exclusions' ' + cat >expect.all <<-\EOF && + Merge-tag-reach (HEAD -> refs/heads/main) + reach (tag: refs/tags/reach, refs/heads/reach) + Merge-tags-octopus-a-and-octopus-b + octopus-b (tag: refs/tags/octopus-b, refs/heads/octopus-b) + octopus-a (tag: refs/tags/octopus-a, refs/heads/octopus-a) + seventh (tag: refs/tags/seventh) + Merge-branch-tangle (refs/hidden/tangle) + Merge-branch-side-early-part-into-tangle (refs/rewritten/merge, refs/heads/tangle) + Merge-branch-main-early-part-into-tangle (refs/prefetch/merge) + tangle-a (tag: refs/tags/tangle-a) + side-2 (tag: refs/tags/side-2, refs/heads/side) + side-1 (tag: refs/tags/side-1) + initial + EOF + + git log --decorate=full --pretty="tformat:%f%d" \ + --simplify-by-decoration \ + --decorate-refs-exclude="heads/octopus*" \ + --decorate-refs="heads" \ + --clear-decorations >actual && + test_cmp expect.all actual && + + cat >expect.filtered <<-\EOF && + Merge-tags-octopus-a-and-octopus-b + octopus-b (refs/heads/octopus-b) + octopus-a (refs/heads/octopus-a) + initial + EOF + + git log --decorate=full --pretty="tformat:%f%d" \ + --simplify-by-decoration \ + --decorate-refs-exclude="heads/octopus" \ + --decorate-refs="heads" \ + --clear-decorations \ + --decorate-refs-exclude="tags/" \ + --decorate-refs="heads/octopus*" >actual && + test_cmp expect.filtered actual +' + test_expect_success 'log.decorate config parsing' ' git log --oneline --decorate=full >expect.full && git log --oneline --decorate=short >expect.short && @@ -2112,9 +2224,9 @@ test_expect_success REFFILES 'log diagnoses bogus HEAD hash' ' test_i18ngrep broken stderr ' -test_expect_success 'log diagnoses bogus HEAD symref' ' +test_expect_success REFFILES 'log diagnoses bogus HEAD symref' ' git init empty && - git --git-dir empty/.git symbolic-ref HEAD refs/heads/invalid.lock && + echo "ref: refs/heads/invalid.lock" > empty/.git/HEAD && test_must_fail git -C empty log 2>stderr && test_i18ngrep broken stderr && test_must_fail git -C empty log --default totally-bogus 2>stderr && @@ -2192,6 +2304,20 @@ test_expect_success 'log --decorate includes all levels of tag annotated tags' ' test_cmp expect actual ' +test_expect_success 'log --decorate does not include things outside filter' ' + reflist="refs/prefetch refs/rebase-merge refs/bundle" && + + for ref in $reflist + do + git update-ref $ref/fake HEAD || return 1 + done && + + git log --decorate=full --oneline >actual && + + # None of the refs are visible: + ! grep /fake actual +' + test_expect_success 'log --end-of-options' ' git update-ref refs/heads/--source HEAD && git log --end-of-options --source >actual && diff --git a/t/t4203-mailmap.sh b/t/t4203-mailmap.sh index 0b2d21ec55..cd1cab3e54 100755 --- a/t/t4203-mailmap.sh +++ b/t/t4203-mailmap.sh @@ -963,4 +963,63 @@ test_expect_success SYMLINKS 'symlinks not respected in-tree' ' test_cmp expect actual ' +test_expect_success 'prepare for cat-file --mailmap' ' + rm -f .mailmap && + git commit --allow-empty -m foo --author="Orig <orig@example.com>" +' + +test_expect_success '--no-use-mailmap disables mailmap in cat-file' ' + test_when_finished "rm .mailmap" && + cat >.mailmap <<-EOF && + A U Thor <author@example.com> Orig <orig@example.com> + EOF + cat >expect <<-EOF && + author Orig <orig@example.com> + EOF + git cat-file --no-use-mailmap commit HEAD >log && + sed -n "/^author /s/\([^>]*>\).*/\1/p" log >actual && + test_cmp expect actual +' + +test_expect_success '--use-mailmap enables mailmap in cat-file' ' + test_when_finished "rm .mailmap" && + cat >.mailmap <<-EOF && + A U Thor <author@example.com> Orig <orig@example.com> + EOF + cat >expect <<-EOF && + author A U Thor <author@example.com> + EOF + git cat-file --use-mailmap commit HEAD >log && + sed -n "/^author /s/\([^>]*>\).*/\1/p" log >actual && + test_cmp expect actual +' + +test_expect_success '--no-mailmap disables mailmap in cat-file for annotated tag objects' ' + test_when_finished "rm .mailmap" && + cat >.mailmap <<-EOF && + Orig <orig@example.com> C O Mitter <committer@example.com> + EOF + cat >expect <<-EOF && + tagger C O Mitter <committer@example.com> + EOF + git tag -a -m "annotated tag" v1 && + git cat-file --no-mailmap -p v1 >log && + sed -n "/^tagger /s/\([^>]*>\).*/\1/p" log >actual && + test_cmp expect actual +' + +test_expect_success '--mailmap enables mailmap in cat-file for annotated tag objects' ' + test_when_finished "rm .mailmap" && + cat >.mailmap <<-EOF && + Orig <orig@example.com> C O Mitter <committer@example.com> + EOF + cat >expect <<-EOF && + tagger Orig <orig@example.com> + EOF + git tag -a -m "annotated tag" v2 && + git cat-file --mailmap -p v2 >log && + sed -n "/^tagger /s/\([^>]*>\).*/\1/p" log >actual && + test_cmp expect actual +' + test_done diff --git a/t/t4207-log-decoration-colors.sh b/t/t4207-log-decoration-colors.sh index 36ac6aff1e..ded33a82e2 100755 --- a/t/t4207-log-decoration-colors.sh +++ b/t/t4207-log-decoration-colors.sh @@ -3,7 +3,7 @@ # Copyright (c) 2010 Nazri Ramliy # -test_description='Test for "git log --decorate" colors' +test_description='test "git log --decorate" colors' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME @@ -17,6 +17,7 @@ test_expect_success setup ' git config color.decorate.remoteBranch red && git config color.decorate.tag "reverse bold yellow" && git config color.decorate.stash magenta && + git config color.decorate.grafted black && git config color.decorate.HEAD cyan && c_reset="<RESET>" && @@ -27,6 +28,7 @@ test_expect_success setup ' c_tag="<BOLD;REVERSE;YELLOW>" && c_stash="<MAGENTA>" && c_HEAD="<CYAN>" && + c_grafted="<BLACK>" && test_commit A && git clone . other && @@ -42,25 +44,79 @@ test_expect_success setup ' git stash save Changes to A.t ' -cat >expected <<EOF -${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD ->\ - ${c_reset}${c_branch}main${c_reset}${c_commit},\ - ${c_reset}${c_tag}tag: v1.0${c_reset}${c_commit},\ - ${c_reset}${c_tag}tag: B${c_reset}${c_commit})${c_reset} B -${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: A1${c_reset}${c_commit},\ - ${c_reset}${c_remoteBranch}other/main${c_reset}${c_commit})${c_reset} A1 -${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_stash}refs/stash${c_reset}${c_commit})${c_reset}\ - On main: Changes to A.t -${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: A${c_reset}${c_commit})${c_reset} A -EOF +cmp_filtered_decorations () { + sed "s/$OID_REGEX/COMMIT_ID/" actual | test_decode_color >filtered && + test_cmp expect filtered +} # We want log to show all, but the second parent to refs/stash is irrelevant # to this test since it does not contain any decoration, hence --first-parent -test_expect_success 'Commit Decorations Colored Correctly' ' - git log --first-parent --abbrev=10 --all --decorate --oneline --color=always | - sed "s/[0-9a-f]\{10,10\}/COMMIT_ID/" | - test_decode_color >out && - test_cmp expected out +test_expect_success 'commit decorations colored correctly' ' + cat >expect <<-EOF && + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD -> \ +${c_reset}${c_branch}main${c_reset}${c_commit}, \ +${c_reset}${c_tag}tag: v1.0${c_reset}${c_commit}, \ +${c_reset}${c_tag}tag: B${c_reset}${c_commit})${c_reset} B +${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: A1${c_reset}${c_commit}, \ +${c_reset}${c_remoteBranch}other/main${c_reset}${c_commit})${c_reset} A1 + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_stash}refs/stash${c_reset}${c_commit})${c_reset} \ +On main: Changes to A.t + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: A${c_reset}${c_commit})${c_reset} A + EOF + + git log --first-parent --no-abbrev --decorate --oneline --color=always --all >actual && + cmp_filtered_decorations +' + +test_expect_success 'test coloring with replace-objects' ' + test_when_finished rm -rf .git/refs/replace* && + test_commit C && + test_commit D && + + git replace HEAD~1 HEAD~2 && + + cat >expect <<-EOF && + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD -> \ +${c_reset}${c_branch}main${c_reset}${c_commit}, \ +${c_reset}${c_tag}tag: D${c_reset}${c_commit})${c_reset} D + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: C${c_reset}${c_commit}, \ +${c_reset}${c_grafted}replaced${c_reset}${c_commit})${c_reset} B + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: A${c_reset}${c_commit})${c_reset} A +EOF + + git log --first-parent --no-abbrev --decorate --oneline --color=always HEAD >actual && + cmp_filtered_decorations && + git replace -d HEAD~1 && + + GIT_REPLACE_REF_BASE=refs/replace2/ git replace HEAD~1 HEAD~2 && + GIT_REPLACE_REF_BASE=refs/replace2/ git log --first-parent \ + --no-abbrev --decorate --oneline --color=always HEAD >actual && + cmp_filtered_decorations +' + +test_expect_success 'test coloring with grafted commit' ' + test_when_finished rm -rf .git/refs/replace* && + + git replace --graft HEAD HEAD~2 && + + cat >expect <<-EOF && + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD -> \ +${c_reset}${c_branch}main${c_reset}${c_commit}, \ +${c_reset}${c_tag}tag: D${c_reset}${c_commit}, \ +${c_reset}${c_grafted}replaced${c_reset}${c_commit})${c_reset} D + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: v1.0${c_reset}${c_commit}, \ +${c_reset}${c_tag}tag: B${c_reset}${c_commit})${c_reset} B + ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_tag}tag: A${c_reset}${c_commit})${c_reset} A + EOF + + git log --first-parent --no-abbrev --decorate --oneline --color=always HEAD >actual && + cmp_filtered_decorations && + git replace -d HEAD && + + GIT_REPLACE_REF_BASE=refs/replace2/ git replace --graft HEAD HEAD~2 && + GIT_REPLACE_REF_BASE=refs/replace2/ git log --first-parent \ + --no-abbrev --decorate --oneline --color=always HEAD >actual && + cmp_filtered_decorations ' test_done diff --git a/t/t4301-merge-tree-write-tree.sh b/t/t4301-merge-tree-write-tree.sh index f091259a55..a243e3c517 100755 --- a/t/t4301-merge-tree-write-tree.sh +++ b/t/t4301-merge-tree-write-tree.sh @@ -2,6 +2,7 @@ test_description='git merge-tree --write-tree' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # This test is ort-specific diff --git a/t/t5315-pack-objects-compression.sh b/t/t5315-pack-objects-compression.sh index 8bacd96275..c80ea9e8b7 100755 --- a/t/t5315-pack-objects-compression.sh +++ b/t/t5315-pack-objects-compression.sh @@ -2,6 +2,7 @@ test_description='pack-object compression configuration' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index 7e040eb1ed..049c5fc8ea 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -812,4 +812,31 @@ test_expect_success 'set up and verify repo with generation data overflow chunk' graph_git_behavior 'generation data overflow chunk repo' repo left right +test_expect_success 'overflow during generation version upgrade' ' + git init overflow-v2-upgrade && + ( + cd overflow-v2-upgrade && + + # This commit will have a date at two seconds past the Epoch, + # and a (v1) generation number of 1, since it is a root commit. + # + # The offset will then be computed as 1-2, which will underflow + # to 2^31, which is greater than the v2 offset small limit of + # 2^31-1. + # + # This is sufficient to need a large offset table for the v2 + # generation numbers. + test_commit --date "@2 +0000" base && + git repack -d && + + # Test that upgrading from generation v1 to v2 correctly + # produces the overflow table. + git -c commitGraph.generationVersion=1 commit-graph write && + git -c commitGraph.generationVersion=2 commit-graph write \ + --changed-paths && + + git rev-list --all + ) +' + test_done diff --git a/t/t5329-pack-objects-cruft.sh b/t/t5329-pack-objects-cruft.sh index 8968f7a08d..303f7a5d84 100755 --- a/t/t5329-pack-objects-cruft.sh +++ b/t/t5329-pack-objects-cruft.sh @@ -29,7 +29,8 @@ basic_cruft_pack_tests () { while read oid do path="$objdir/$(test_oid_to_path "$oid")" && - printf "%s %d\n" "$oid" "$(test-tool chmtime --get "$path")" + printf "%s %d\n" "$oid" "$(test-tool chmtime --get "$path")" || + echo "object list generation failed for $oid" done | sort -k1 ) >expect && @@ -232,7 +233,7 @@ test_expect_success 'cruft tags rescue tagged objects' ' while read oid do test-tool chmtime -1000 \ - "$objdir/$(test_oid_to_path $oid)" + "$objdir/$(test_oid_to_path $oid)" || exit 1 done <objects && test-tool chmtime -500 \ @@ -272,7 +273,7 @@ test_expect_success 'cruft commits rescue parents, trees' ' while read object do test-tool chmtime -1000 \ - "$objdir/$(test_oid_to_path $object)" + "$objdir/$(test_oid_to_path $object)" || exit 1 done <objects && test-tool chmtime +500 "$objdir/$(test_oid_to_path \ $(git rev-parse HEAD))" && @@ -345,7 +346,7 @@ test_expect_success 'expired objects are pruned' ' while read object do test-tool chmtime -1000 \ - "$objdir/$(test_oid_to_path $object)" + "$objdir/$(test_oid_to_path $object)" || exit 1 done <objects && keep="$(basename "$(ls $packdir/pack-*.pack)")" && diff --git a/t/t5351-unpack-large-objects.sh b/t/t5351-unpack-large-objects.sh index 8ce8aa3b14..8c8af99b84 100755 --- a/t/t5351-unpack-large-objects.sh +++ b/t/t5351-unpack-large-objects.sh @@ -5,6 +5,7 @@ test_description='git unpack-objects with large objects' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh prepare_dest () { @@ -48,13 +49,39 @@ test_expect_success 'unpack big object in stream' ' test_dir_is_empty dest.git/objects/pack ' +check_fsync_events () { + local trace="$1" && + shift && + + cat >expect && + sed -n \ + -e '/^{"event":"data",.*"category":"fsync",/ { + s/.*"category":"fsync",//; + s/}$//; + p; + }' \ + <"$trace" >actual && + test_cmp expect actual +} + BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch' test_expect_success 'unpack big object in stream (core.fsyncmethod=batch)' ' prepare_dest 1m && GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + GIT_TEST_FSYNC=true \ git -C dest.git $BATCH_CONFIGURATION unpack-objects <pack-$PACK.pack && - grep fsync/hardware-flush trace2.txt && + if grep "core.fsyncMethod = batch is unsupported" trace2.txt + then + flush_count=7 + else + flush_count=1 + fi && + check_fsync_events trace2.txt <<-EOF && + "key":"fsync/writeout-only","value":"6" + "key":"fsync/hardware-flush","value":"$flush_count" + EOF + test_dir_is_empty dest.git/objects/pack && git -C dest.git cat-file --batch-check="%(objectname)" <obj-list >current && cmp obj-list current @@ -67,7 +94,7 @@ test_expect_success 'do not unpack existing large objects' ' # The destination came up with the exact same pack... DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) && - test_cmp pack-$PACK.pack $DEST_PACK && + cmp pack-$PACK.pack $DEST_PACK && # ...and wrote no loose objects test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*" diff --git a/t/t5402-post-merge-hook.sh b/t/t5402-post-merge-hook.sh index 915af2de95..46ebdfbeeb 100755 --- a/t/t5402-post-merge-hook.sh +++ b/t/t5402-post-merge-hook.sh @@ -7,6 +7,7 @@ test_description='Test the post-merge hook.' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index ee6d2dde9f..d18f2823d8 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -407,6 +407,7 @@ test_expect_success 'in_vain not triggered before first ACK' ' ' test_expect_success 'in_vain resetted upon ACK' ' + test_when_finished rm -f log trace2 && rm -rf myserver myclient && git init myserver && @@ -432,7 +433,8 @@ test_expect_success 'in_vain resetted upon ACK' ' # first. The 256th commit is common between the client and the server, # and should reset in_vain. This allows negotiation to continue until # the client reports that first_anotherbranch_commit is common. - git -C myclient fetch --progress origin main 2>log && + GIT_TRACE2_EVENT="$(pwd)/trace2" git -C myclient fetch --progress origin main 2>log && + grep \"key\":\"total_rounds\",\"value\":\"6\" trace2 && test_i18ngrep "Total 3 " log ' diff --git a/t/t5503-tagfollow.sh b/t/t5503-tagfollow.sh index 195fc64dd4..5ebbaa4896 100755 --- a/t/t5503-tagfollow.sh +++ b/t/t5503-tagfollow.sh @@ -5,6 +5,7 @@ test_description='test automatic tag following' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # End state of the repository: diff --git a/t/t5504-fetch-receive-strict.sh b/t/t5504-fetch-receive-strict.sh index b0b795aca9..ac4099ca89 100755 --- a/t/t5504-fetch-receive-strict.sh +++ b/t/t5504-fetch-receive-strict.sh @@ -352,4 +352,21 @@ test_expect_success \ grep "Cannot demote unterminatedheader" act ' +test_expect_success 'badFilemode is not a strict error' ' + git init --bare badmode.git && + tree=$( + cd badmode.git && + blob=$(echo blob | git hash-object -w --stdin | hex2oct) && + printf "123456 foo\0${blob}" | + git hash-object -t tree --stdin -w --literally + ) && + + rm -rf dst.git && + git init --bare dst.git && + git -C dst.git config transfer.fsckObjects true && + + git -C badmode.git push ../dst.git $tree:refs/tags/tree 2>err && + grep "$tree: badFilemode" err +' + test_done diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index f3356f9ea8..3211002d46 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -200,7 +200,10 @@ test_expect_success 'push with negotiation' ' test_commit -C testrepo unrelated_commit && git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit && test_when_finished "rm event" && - GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main && + GIT_TRACE2_EVENT="$(pwd)/event" \ + git -c protocol.version=2 -c push.negotiate=1 \ + push testrepo refs/heads/main:refs/remotes/origin/main && + grep \"key\":\"total_rounds\",\"value\":\"1\" event && grep_wrote 2 event # 1 commit, 1 tree ' @@ -224,7 +227,10 @@ test_expect_success 'push with negotiation does not attempt to fetch submodules' git push testrepo $the_first_commit:refs/remotes/origin/first_commit && test_commit -C testrepo unrelated_commit && git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit && - git -c submodule.recurse=true -c protocol.version=2 -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main 2>err && + GIT_TRACE2_EVENT="$(pwd)/event" git -c submodule.recurse=true \ + -c protocol.version=2 -c push.negotiate=1 \ + push testrepo refs/heads/main:refs/remotes/origin/main 2>err && + grep \"key\":\"total_rounds\",\"value\":\"1\" event && ! grep "Fetching submodule" err ' diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index 245532df88..6a38294a47 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -181,8 +181,8 @@ test_expect_success 'no-op half-auth fetch does not require a password' ' # This is not possible with protocol v2, since both objects and refs # are obtained from the "git-upload-pack" path. A solution to this is # to teach the server and client to be able to inline ls-refs requests - # as an Extra Parameter (see pack-protocol.txt), so that "info/refs" - # can serve refs, just like it does in protocol v0. + # as an Extra Parameter (see "git help gitformat-pack-protocol"), so that + # "info/refs" can serve refs, just like it does in protocol v0. GIT_TEST_PROTOCOL_VERSION=0 git --git-dir=half-auth fetch && expect_askpass none ' diff --git a/t/t5557-http-get.sh b/t/t5557-http-get.sh new file mode 100755 index 0000000000..76a4bbd16a --- /dev/null +++ b/t/t5557-http-get.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description='test downloading a file by URL' + +TEST_PASSES_SANITIZE_LEAK=true + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'get by URL: 404' ' + test_when_finished "rm -f file.temp" && + url="$HTTPD_URL/none.txt" && + cat >input <<-EOF && + capabilities + get $url file1 + EOF + + test_must_fail git remote-http $url <input 2>err && + test_path_is_missing file1 && + grep "failed to download file at URL" err +' + +test_expect_success 'get by URL: 200' ' + echo data >"$HTTPD_DOCUMENT_ROOT_PATH/exists.txt" && + + url="$HTTPD_URL/exists.txt" && + cat >input <<-EOF && + capabilities + get $url file2 + + EOF + + git remote-http $url <input && + test_cmp "$HTTPD_DOCUMENT_ROOT_PATH/exists.txt" file2 +' + +test_done diff --git a/t/t5558-clone-bundle-uri.sh b/t/t5558-clone-bundle-uri.sh new file mode 100755 index 0000000000..ad666a2d28 --- /dev/null +++ b/t/t5558-clone-bundle-uri.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +test_description='test fetching bundles with --bundle-uri' + +. ./test-lib.sh + +test_expect_success 'fail to clone from non-existent file' ' + test_when_finished rm -rf test && + git clone --bundle-uri="$(pwd)/does-not-exist" . test 2>err && + grep "failed to download bundle from URI" err +' + +test_expect_success 'fail to clone from non-bundle file' ' + test_when_finished rm -rf test && + echo bogus >bogus && + git clone --bundle-uri="$(pwd)/bogus" . test 2>err && + grep "is not a bundle" err +' + +test_expect_success 'create bundle' ' + git init clone-from && + git -C clone-from checkout -b topic && + test_commit -C clone-from A && + test_commit -C clone-from B && + git -C clone-from bundle create B.bundle topic +' + +test_expect_success 'clone with path bundle' ' + git clone --bundle-uri="clone-from/B.bundle" \ + clone-from clone-path && + git -C clone-path rev-parse refs/bundles/topic >actual && + git -C clone-from rev-parse topic >expect && + test_cmp expect actual +' + +test_expect_success 'clone with file:// bundle' ' + git clone --bundle-uri="file://$(pwd)/clone-from/B.bundle" \ + clone-from clone-file && + git -C clone-file rev-parse refs/bundles/topic >actual && + git -C clone-from rev-parse topic >expect && + test_cmp expect actual +' + +######################################################################### +# HTTP tests begin here + +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'fail to fetch from non-existent HTTP URL' ' + test_when_finished rm -rf test && + git clone --bundle-uri="$HTTPD_URL/does-not-exist" . test 2>err && + grep "failed to download bundle from URI" err +' + +test_expect_success 'fail to fetch from non-bundle HTTP URL' ' + test_when_finished rm -rf test && + echo bogus >"$HTTPD_DOCUMENT_ROOT_PATH/bogus" && + git clone --bundle-uri="$HTTPD_URL/bogus" . test 2>err && + grep "is not a bundle" err +' + +test_expect_success 'clone HTTP bundle' ' + cp clone-from/B.bundle "$HTTPD_DOCUMENT_ROOT_PATH/B.bundle" && + + git clone --no-local --mirror clone-from \ + "$HTTPD_DOCUMENT_ROOT_PATH/fetch.git" && + + git clone --bundle-uri="$HTTPD_URL/B.bundle" \ + "$HTTPD_URL/smart/fetch.git" clone-http && + git -C clone-http rev-parse refs/bundles/topic >actual && + git -C clone-from rev-parse topic >expect && + test_cmp expect actual && + + test_config -C clone-http log.excludedecoration refs/bundle/ +' + +# Do not add tests here unless they use the HTTP server, as they will +# not run unless the HTTP dependencies exist. + +test_done diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index cf3be0584f..2e57de9c12 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -743,7 +743,11 @@ test_expect_success 'batch missing blob request during checkout' ' # Ensure that there is only one negotiation by checking that there is # only "done" line sent. ("done" marks the end of negotiation.) - GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ && + GIT_TRACE_PACKET="$(pwd)/trace" \ + GIT_TRACE2_EVENT="$(pwd)/trace2_event" \ + git -C client -c trace2.eventNesting=5 checkout HEAD^ && + grep \"key\":\"total_rounds\",\"value\":\"1\" trace2_event >trace_lines && + test_line_count = 1 trace_lines && grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' diff --git a/t/t5606-clone-options.sh b/t/t5606-clone-options.sh index 8f676d6b0c..f6bb02ab94 100755 --- a/t/t5606-clone-options.sh +++ b/t/t5606-clone-options.sh @@ -58,6 +58,14 @@ test_expect_success 'disallows --bare with --separate-git-dir' ' ' +test_expect_success 'disallows --bundle-uri with shallow options' ' + for option in --depth=1 --shallow-since=01-01-2000 --shallow-exclude=HEAD + do + test_must_fail git clone --bundle-uri=bundle $option from to 2>err && + grep "bundle-uri is incompatible" err || return 1 + done +' + test_expect_success 'reject cloning shallow repository' ' test_when_finished "rm -rf repo" && test_must_fail git clone --reject-shallow shallow-repo out 2>err && diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 4a3778d04a..9aeacc2f6a 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -49,6 +49,13 @@ test_expect_success 'do partial clone 1' ' test "$(git -C pc1 config --local remote.origin.partialclonefilter)" = "blob:none" ' +test_expect_success 'rev-list --missing=allow-promisor on partial clone' ' + git -C pc1 rev-list --objects --missing=allow-promisor HEAD >actual && + git -C pc1 rev-list --objects --missing=print HEAD >expect.raw && + grep -v "^?" expect.raw >expect && + test_cmp expect actual +' + test_expect_success 'verify that .promisor file contains refs fetched' ' ls pc1/.git/objects/pack/pack-*.promisor >promisorlist && test_line_count = 1 promisorlist && diff --git a/t/t5703-upload-pack-ref-in-want.sh b/t/t5703-upload-pack-ref-in-want.sh index 9d6cd7d986..df74f80061 100755 --- a/t/t5703-upload-pack-ref-in-want.sh +++ b/t/t5703-upload-pack-ref-in-want.sh @@ -229,14 +229,16 @@ test_expect_success 'setup repos for fetching with ref-in-want tests' ' ' test_expect_success 'fetching with exact OID' ' - test_when_finished "rm -f log" && + test_when_finished "rm -f log trace2" && rm -rf local && cp -r "$LOCAL_PRISTINE" local && oid=$(git -C "$REPO" rev-parse d) && - GIT_TRACE_PACKET="$(pwd)/log" git -C local fetch origin \ + GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE2_EVENT="$(pwd)/trace2" \ + git -C local fetch origin \ "$oid":refs/heads/actual && + grep \"key\":\"total_rounds\",\"value\":\"2\" trace2 && git -C "$REPO" rev-parse "d" >expected && git -C local rev-parse refs/heads/actual >actual && test_cmp expected actual && diff --git a/t/t5812-proto-disable-http.sh b/t/t5812-proto-disable-http.sh index af8772fada..d8da5f58d1 100755 --- a/t/t5812-proto-disable-http.sh +++ b/t/t5812-proto-disable-http.sh @@ -16,7 +16,7 @@ test_expect_success 'create git-accessible repo' ' test_proto "smart http" http "$HTTPD_URL/smart/repo.git" -test_expect_success 'curl redirects respect whitelist' ' +test_expect_success 'http(s) transport respects GIT_ALLOW_PROTOCOL' ' test_must_fail env GIT_ALLOW_PROTOCOL=http:https \ GIT_SMART_HTTP=0 \ git clone "$HTTPD_URL/ftp-redir/repo.git" 2>stderr && diff --git a/t/t5815-submodule-protos.sh b/t/t5815-submodule-protos.sh index 06f55a1b8a..4d5956cc18 100755 --- a/t/t5815-submodule-protos.sh +++ b/t/t5815-submodule-protos.sh @@ -1,6 +1,6 @@ #!/bin/sh -test_description='test protocol whitelisting with submodules' +test_description='test protocol filtering with submodules' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-proto-disable.sh @@ -36,7 +36,7 @@ test_expect_success 'update of ext not allowed' ' test_must_fail git -C dst submodule update ext-module ' -test_expect_success 'user can override whitelist' ' +test_expect_success 'user can filter protocols with GIT_ALLOW_PROTOCOL' ' GIT_ALLOW_PROTOCOL=ext git -C dst submodule update ext-module ' diff --git a/t/t6019-rev-list-ancestry-path.sh b/t/t6019-rev-list-ancestry-path.sh index af57a04b7f..738da23628 100755 --- a/t/t6019-rev-list-ancestry-path.sh +++ b/t/t6019-rev-list-ancestry-path.sh @@ -8,8 +8,13 @@ test_description='--ancestry-path' # / \ # A-------K---------------L--M # -# D..M == E F G H I J K L M -# --ancestry-path D..M == E F H I J L M +# D..M == E F G H I J K L M +# --ancestry-path D..M == E F H I J L M +# --ancestry-path=F D..M == E F J L M +# --ancestry-path=G D..M == G H I J L M +# --ancestry-path=H D..M == E G H I J L M +# --ancestry-path=K D..M == K L M +# --ancestry-path=K --ancestry-path=F D..M == E F J K L M # # D..M -- M.t == M # --ancestry-path D..M -- M.t == M @@ -50,73 +55,41 @@ test_expect_success setup ' test_commit M ' -test_expect_success 'rev-list D..M' ' - test_write_lines E F G H I J K L M >expect && - git rev-list --format=%s D..M | - sed -e "/^commit /d" | - sort >actual && - test_cmp expect actual -' - -test_expect_success 'rev-list --ancestry-path D..M' ' - test_write_lines E F H I J L M >expect && - git rev-list --ancestry-path --format=%s D..M | - sed -e "/^commit /d" | - sort >actual && - test_cmp expect actual -' - -test_expect_success 'rev-list D..M -- M.t' ' - echo M >expect && - git rev-list --format=%s D..M -- M.t | - sed -e "/^commit /d" >actual && - test_cmp expect actual -' - -test_expect_success 'rev-list --ancestry-path D..M -- M.t' ' - echo M >expect && - git rev-list --ancestry-path --format=%s D..M -- M.t | - sed -e "/^commit /d" >actual && - test_cmp expect actual -' +test_ancestry () { + args=$1 + expected=$2 + test_expect_success "log $args" " + test_write_lines $expected >expect && + git log --format=%s $args >raw && + + if test -n \"$expected\" + then + sort raw >actual && + test_cmp expect actual + else + test_must_be_empty raw + fi + " +} -test_expect_success 'rev-list F...I' ' - test_write_lines F G H I >expect && - git rev-list --format=%s F...I | - sed -e "/^commit /d" | - sort >actual && - test_cmp expect actual -' +test_ancestry "D..M" "E F G H I J K L M" -test_expect_success 'rev-list --ancestry-path F...I' ' - test_write_lines F H I >expect && - git rev-list --ancestry-path --format=%s F...I | - sed -e "/^commit /d" | - sort >actual && - test_cmp expect actual -' +test_ancestry "--ancestry-path D..M" "E F H I J L M" +test_ancestry "--ancestry-path=F D..M" "E F J L M" +test_ancestry "--ancestry-path=G D..M" "G H I J L M" +test_ancestry "--ancestry-path=H D..M" "E G H I J L M" +test_ancestry "--ancestry-path=K D..M" "K L M" +test_ancestry "--ancestry-path=F --ancestry-path=K D..M" "E F J K L M" -# G.t is dropped in an "-s ours" merge -test_expect_success 'rev-list G..M -- G.t' ' - git rev-list --format=%s G..M -- G.t | - sed -e "/^commit /d" >actual && - test_must_be_empty actual -' +test_ancestry "D..M -- M.t" "M" +test_ancestry "--ancestry-path D..M -- M.t" "M" -test_expect_success 'rev-list --ancestry-path G..M -- G.t' ' - echo L >expect && - git rev-list --ancestry-path --format=%s G..M -- G.t | - sed -e "/^commit /d" >actual && - test_cmp expect actual -' +test_ancestry "F...I" "F G H I" +test_ancestry "--ancestry-path F...I" "F H I" -test_expect_success 'rev-list --ancestry-path --simplify-merges G^..M -- G.t' ' - test_write_lines G L >expect && - git rev-list --ancestry-path --simplify-merges --format=%s G^..M -- G.t | - sed -e "/^commit /d" | - sort >actual && - test_cmp expect actual -' +test_ancestry "G..M -- G.t" "" +test_ancestry "--ancestry-path G..M -- G.t" "L" +test_ancestry "--ancestry-path --simplify-merges G^..M -- G.t" "G L" # b---bc # / \ / diff --git a/t/t6102-rev-list-unexpected-objects.sh b/t/t6102-rev-list-unexpected-objects.sh index cf0195e826..4a9a4436e2 100755 --- a/t/t6102-rev-list-unexpected-objects.sh +++ b/t/t6102-rev-list-unexpected-objects.sh @@ -17,7 +17,7 @@ test_expect_success 'setup unexpected non-blob entry' ' broken_tree="$(git hash-object -w --literally -t tree broken-tree)" ' -test_expect_success !SANITIZE_LEAK 'TODO (should fail!): traverse unexpected non-blob entry (lone)' ' +test_expect_success 'TODO (should fail!): traverse unexpected non-blob entry (lone)' ' sed "s/Z$//" >expect <<-EOF && $broken_tree Z $tree foo @@ -121,7 +121,7 @@ test_expect_success 'setup unexpected non-blob tag' ' tag=$(git hash-object -w --literally -t tag broken-tag) ' -test_expect_success !SANITIZE_LEAK 'TODO (should fail!): traverse unexpected non-blob tag (lone)' ' +test_expect_success 'TODO (should fail!): traverse unexpected non-blob tag (lone)' ' git rev-list --objects $tag ' diff --git a/t/t6115-rev-list-du.sh b/t/t6115-rev-list-du.sh index b4aef32b71..d59111dede 100755 --- a/t/t6115-rev-list-du.sh +++ b/t/t6115-rev-list-du.sh @@ -48,4 +48,26 @@ check_du HEAD check_du --objects HEAD check_du --objects HEAD^..HEAD +# As mentioned above, don't use hardcode sizes as actual size, but use the +# output from git cat-file. +test_expect_success 'rev-list --disk-usage=human' ' + git rev-list --objects HEAD --disk-usage=human >actual && + disk_usage_slow --objects HEAD >actual_size && + grep "$(cat actual_size) bytes" actual +' + +test_expect_success 'rev-list --disk-usage=human with bitmaps' ' + git rev-list --objects HEAD --use-bitmap-index --disk-usage=human >actual && + disk_usage_slow --objects HEAD >actual_size && + grep "$(cat actual_size) bytes" actual +' + +test_expect_success 'rev-list use --disk-usage unproperly' ' + test_must_fail git rev-list --objects HEAD --disk-usage=typo 2>err && + cat >expect <<-\EOF && + fatal: invalid value for '\''--disk-usage=<format>'\'': '\''typo'\'', the only allowed format is '\''human'\'' + EOF + test_cmp err expect +' + test_done diff --git a/t/t6402-merge-rename.sh b/t/t6402-merge-rename.sh index 3a32b1a45c..772238e582 100755 --- a/t/t6402-merge-rename.sh +++ b/t/t6402-merge-rename.sh @@ -210,7 +210,7 @@ test_expect_success 'updated working tree file should prevent the merge' ' echo >>M one line addition && cat M >M.saved && git update-index M && - test_expect_code 128 git pull --no-rebase . yellow && + test_expect_code 2 git pull --no-rebase . yellow && test_cmp M M.saved && rm -f M.saved ' diff --git a/t/t6404-recursive-merge.sh b/t/t6404-recursive-merge.sh index b8735c6db4..36215518b6 100755 --- a/t/t6404-recursive-merge.sh +++ b/t/t6404-recursive-merge.sh @@ -4,6 +4,7 @@ test_description='Test merge without common ancestors' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # This scenario is based on a real-world repository of Shawn Pearce. diff --git a/t/t6405-merge-symlinks.sh b/t/t6405-merge-symlinks.sh index 7435fce71e..29e2b25ce5 100755 --- a/t/t6405-merge-symlinks.sh +++ b/t/t6405-merge-symlinks.sh @@ -11,6 +11,7 @@ if core.symlinks is false.' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'setup' ' diff --git a/t/t6407-merge-binary.sh b/t/t6407-merge-binary.sh index 0753fc95f4..e8a28717ce 100755 --- a/t/t6407-merge-binary.sh +++ b/t/t6407-merge-binary.sh @@ -5,7 +5,6 @@ test_description='ask merge-recursive to merge binary files' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME -TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t6408-merge-up-to-date.sh b/t/t6408-merge-up-to-date.sh index 7763c1ba98..8a1ba6d23a 100755 --- a/t/t6408-merge-up-to-date.sh +++ b/t/t6408-merge-up-to-date.sh @@ -2,6 +2,7 @@ test_description='merge fast-forward and up to date' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t6411-merge-filemode.sh b/t/t6411-merge-filemode.sh index 6ae2489286..b6182723aa 100755 --- a/t/t6411-merge-filemode.sh +++ b/t/t6411-merge-filemode.sh @@ -4,6 +4,7 @@ test_description='merge: handle file mode' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'set up mode change in one branch' ' diff --git a/t/t6413-merge-crlf.sh b/t/t6413-merge-crlf.sh index affea255fe..b4f4a313f4 100755 --- a/t/t6413-merge-crlf.sh +++ b/t/t6413-merge-crlf.sh @@ -11,6 +11,7 @@ test_description='merge conflict in crlf repo GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t6424-merge-unrelated-index-changes.sh b/t/t6424-merge-unrelated-index-changes.sh index b6e424a427..a61f20c22f 100755 --- a/t/t6424-merge-unrelated-index-changes.sh +++ b/t/t6424-merge-unrelated-index-changes.sh @@ -114,6 +114,39 @@ test_expect_success 'resolve, non-trivial' ' test_path_is_missing .git/MERGE_HEAD ' +test_expect_success 'resolve, trivial, related file removed' ' + git reset --hard && + git checkout B^0 && + + git rm a && + test_path_is_missing a && + + test_must_fail git merge -s resolve C^0 && + + test_path_is_missing a && + test_path_is_missing .git/MERGE_HEAD +' + +test_expect_success 'resolve, non-trivial, related file removed' ' + git reset --hard && + git checkout B^0 && + + git rm a && + test_path_is_missing a && + + # We also ask for recursive in order to turn off the "allow_trivial" + # setting in builtin/merge.c, and ensure that resolve really does + # correctly fail the merge (I guess this also tests that recursive + # correctly fails the merge, but the main thing we are attempting + # to test here is resolve and are just using the side effect of + # adding recursive to ensure that resolve is actually tested rather + # than the trivial merge codepath) + test_must_fail git merge -s resolve -s recursive D^0 && + + test_path_is_missing a && + test_path_is_missing .git/MERGE_HEAD +' + test_expect_success 'recursive' ' git reset --hard && git checkout B^0 && @@ -242,4 +275,36 @@ test_expect_success 'subtree' ' test_path_is_missing .git/MERGE_HEAD ' +test_expect_success 'avoid failure due to stat-dirty files' ' + git reset --hard && + git checkout B^0 && + + # Make "a" be stat-dirty + test-tool chmtime =+1 a && + + # stat-dirty file should not prevent stash creation in builtin/merge.c + git merge -s resolve -s recursive D^0 +' + +test_expect_success 'with multiple strategies, recursive or ort failure do not early abort' ' + git reset --hard && + git checkout B^0 && + + test_seq 0 10 >a && + git add a && + git rev-parse :a >expect && + + sane_unset GIT_TEST_MERGE_ALGORITHM && + test_must_fail git merge -s recursive -s ort -s octopus C^0 >output 2>&1 && + + grep "Trying merge strategy recursive..." output && + grep "Trying merge strategy ort..." output && + grep "Trying merge strategy octopus..." output && + grep "No merge strategy handled the merge." output && + + # Changes to "a" should remain staged + git rev-parse :a >actual && + test_cmp expect actual +' + test_done diff --git a/t/t6425-merge-rename-delete.sh b/t/t6425-merge-rename-delete.sh index 459b431a60..93cd2869b1 100755 --- a/t/t6425-merge-rename-delete.sh +++ b/t/t6425-merge-rename-delete.sh @@ -4,6 +4,7 @@ test_description='Merge-recursive rename/delete conflict message' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'rename/delete' ' diff --git a/t/t6429-merge-sequence-rename-caching.sh b/t/t6429-merge-sequence-rename-caching.sh index e1ce919916..650b3cd14f 100755 --- a/t/t6429-merge-sequence-rename-caching.sh +++ b/t/t6429-merge-sequence-rename-caching.sh @@ -725,7 +725,7 @@ test_expect_success 'avoid assuming we detected renames' ' mkdir unrelated && for i in $(test_seq 1 10) do - >unrelated/$i + >unrelated/$i || exit 1 done && test_seq 2 10 >numbers && test_seq 12 20 >values && diff --git a/t/t6431-merge-criscross.sh b/t/t6431-merge-criscross.sh index 3824756a02..3fe14cd73e 100755 --- a/t/t6431-merge-criscross.sh +++ b/t/t6431-merge-criscross.sh @@ -2,6 +2,7 @@ test_description='merge-recursive backend test' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # A <- create some files diff --git a/t/t6437-submodule-merge.sh b/t/t6437-submodule-merge.sh index c253bf759a..d5f3e0fed6 100755 --- a/t/t6437-submodule-merge.sh +++ b/t/t6437-submodule-merge.sh @@ -103,8 +103,25 @@ test_expect_success 'setup for merge search' ' echo "file-c" > file-c && git add file-c && git commit -m "sub-c") && - git commit -a -m "c" && + git commit -a -m "c") +' +test_expect_success 'merging should conflict for non fast-forward' ' + test_when_finished "git -C merge-search reset --hard" && + (cd merge-search && + git checkout -b test-nonforward-a b && + if test "$GIT_TEST_MERGE_ALGORITHM" = ort + then + test_must_fail git merge c >actual && + sub_expect="go to submodule (sub), and either merge commit $(git -C sub rev-parse --short sub-c)" && + grep "$sub_expect" actual + else + test_must_fail git merge c 2> actual + fi) +' + +test_expect_success 'finish setup for merge-search' ' + (cd merge-search && git checkout -b d a && (cd sub && git checkout -b sub-d sub-b && @@ -129,14 +146,16 @@ test_expect_success 'merge with one side as a fast-forward of the other' ' test_cmp expect actual) ' -test_expect_success 'merging should conflict for non fast-forward' ' +test_expect_success 'merging should conflict for non fast-forward (resolution exists)' ' (cd merge-search && - git checkout -b test-nonforward b && + git checkout -b test-nonforward-b b && (cd sub && git rev-parse --short sub-d > ../expect) && if test "$GIT_TEST_MERGE_ALGORITHM" = ort then - test_must_fail git merge c >actual + test_must_fail git merge c >actual && + sub_expect="go to submodule (sub), and either merge commit $(git -C sub rev-parse --short sub-c)" && + grep "$sub_expect" actual else test_must_fail git merge c 2> actual fi && @@ -161,7 +180,9 @@ test_expect_success 'merging should fail for ambiguous common parent' ' ) && if test "$GIT_TEST_MERGE_ALGORITHM" = ort then - test_must_fail git merge c >actual + test_must_fail git merge c >actual && + sub_expect="go to submodule (sub), and either merge commit $(git -C sub rev-parse --short sub-c)" && + grep "$sub_expect" actual else test_must_fail git merge c 2> actual fi && @@ -205,7 +226,12 @@ test_expect_success 'merging should fail for changes that are backwards' ' git commit -a -m "f" && git checkout -b test-backward e && - test_must_fail git merge f) + test_must_fail git merge f >actual && + if test "$GIT_TEST_MERGE_ALGORITHM" = ort + then + sub_expect="go to submodule (sub), and either merge commit $(git -C sub rev-parse --short sub-d)" && + grep "$sub_expect" actual + fi) ' @@ -476,4 +502,44 @@ test_expect_failure 'directory/submodule conflict; merge --abort works afterward ) ' +# Setup: +# - Submodule has 2 commits: a and b +# - Superproject branch 'a' adds and commits submodule pointing to 'commit a' +# - Superproject branch 'b' adds and commits submodule pointing to 'commit b' +# If these two branches are now merged, there is no merge base +test_expect_success 'setup for null merge base' ' + mkdir no-merge-base && + (cd no-merge-base && + git init && + mkdir sub && + (cd sub && + git init && + echo "file-a" > file-a && + git add file-a && + git commit -m "commit a") && + git commit --allow-empty -m init && + git branch init && + git checkout -b a init && + git add sub && + git commit -m "a" && + git switch main && + (cd sub && + echo "file-b" > file-b && + git add file-b && + git commit -m "commit b")) +' + +test_expect_success 'merging should fail with no merge base' ' + (cd no-merge-base && + git checkout -b b init && + git add sub && + git commit -m "b" && + test_must_fail git merge a >actual && + if test "$GIT_TEST_MERGE_ALGORITHM" = ort + then + sub_expect="go to submodule (sub), and either merge commit $(git -C sub rev-parse --short HEAD^1)" && + grep "$sub_expect" actual + fi) +' + test_done diff --git a/t/t6439-merge-co-error-msgs.sh b/t/t6439-merge-co-error-msgs.sh index 5bfb027099..52cf0c8769 100755 --- a/t/t6439-merge-co-error-msgs.sh +++ b/t/t6439-merge-co-error-msgs.sh @@ -47,6 +47,7 @@ test_expect_success 'untracked files overwritten by merge (fast and non-fast for export GIT_MERGE_VERBOSITY && test_must_fail git merge branch 2>out2 ) && + echo "Merge with strategy ${GIT_TEST_MERGE_ALGORITHM:-ort} failed." >>expect && test_cmp out2 expect && git reset --hard HEAD^ ' diff --git a/t/t7007-show.sh b/t/t7007-show.sh index d6cc69e0f2..f908a4d1ab 100755 --- a/t/t7007-show.sh +++ b/t/t7007-show.sh @@ -2,6 +2,7 @@ test_description='git show' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t7060-wtstatus.sh b/t/t7060-wtstatus.sh index 0f4344c55e..aaeb4a5334 100755 --- a/t/t7060-wtstatus.sh +++ b/t/t7060-wtstatus.sh @@ -5,6 +5,7 @@ test_description='basic work tree status reporting' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t7062-wtstatus-ignorecase.sh b/t/t7062-wtstatus-ignorecase.sh index 73709dbeee..caf372a3d4 100755 --- a/t/t7062-wtstatus-ignorecase.sh +++ b/t/t7062-wtstatus-ignorecase.sh @@ -2,6 +2,7 @@ test_description='git-status with core.ignorecase=true' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'status with hash collisions' ' diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index f5050b75b2..8929ef481f 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -986,4 +986,9 @@ test_expect_success '"status" after file replacement should be clean with UC=fal status_is_clean ' +test_expect_success 'empty repo (no index) and core.untrackedCache' ' + git init emptyrepo && + git -C emptyrepo -c core.untrackedCache=true write-tree +' + test_done diff --git a/t/t7110-reset-merge.sh b/t/t7110-reset-merge.sh index 3d62e10b53..eb881be95b 100755 --- a/t/t7110-reset-merge.sh +++ b/t/t7110-reset-merge.sh @@ -5,6 +5,7 @@ test_description='Tests for "git reset" with "--merge" and "--keep" options' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success setup ' diff --git a/t/t7111-reset-table.sh b/t/t7111-reset-table.sh index ce421ad5ac..78f25c1c7e 100755 --- a/t/t7111-reset-table.sh +++ b/t/t7111-reset-table.sh @@ -5,6 +5,7 @@ test_description='Tests to check that "reset" options follow a known table' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh diff --git a/t/t7402-submodule-rebase.sh b/t/t7402-submodule-rebase.sh index 8e32f19007..ebeca12a71 100755 --- a/t/t7402-submodule-rebase.sh +++ b/t/t7402-submodule-rebase.sh @@ -104,7 +104,7 @@ test_expect_success 'rebasing submodule that should conflict' ' test_tick && git commit -m fourth && - test_must_fail git rebase --onto HEAD^^ HEAD^ HEAD^0 && + test_must_fail git rebase --onto HEAD^^ HEAD^ HEAD^0 >actual_output && git ls-files -s submodule >actual && ( cd submodule && @@ -112,7 +112,12 @@ test_expect_success 'rebasing submodule that should conflict' ' echo "160000 $(git rev-parse HEAD^^) 2 submodule" && echo "160000 $(git rev-parse HEAD) 3 submodule" ) >expect && - test_cmp expect actual + test_cmp expect actual && + if test "$GIT_TEST_MERGE_ALGORITHM" = ort + then + sub_expect="go to submodule (submodule), and either merge commit $(git -C submodule rev-parse --short HEAD^0)" && + grep "$sub_expect" actual_output + fi ' test_done diff --git a/t/t7503-pre-commit-and-pre-merge-commit-hooks.sh b/t/t7503-pre-commit-and-pre-merge-commit-hooks.sh index ad1eb64ba0..aa004b70a8 100755 --- a/t/t7503-pre-commit-and-pre-merge-commit-hooks.sh +++ b/t/t7503-pre-commit-and-pre-merge-commit-hooks.sh @@ -5,6 +5,7 @@ test_description='pre-commit and pre-merge-commit hooks' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'root commit' ' diff --git a/t/t7607-merge-state.sh b/t/t7607-merge-state.sh new file mode 100755 index 0000000000..89a62ac53b --- /dev/null +++ b/t/t7607-merge-state.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description="Test that merge state is as expected after failed merge" + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +. ./test-lib.sh + +test_expect_success 'Ensure we restore original state if no merge strategy handles it' ' + test_commit --no-tag "Initial" base base && + + for b in branch1 branch2 branch3 + do + git checkout -b $b main && + test_commit --no-tag "Change on $b" base $b || return 1 + done && + + git checkout branch1 && + # This is a merge that octopus cannot handle. Note, that it does not + # just hit conflicts, it completely fails and says that it cannot + # handle this type of merge. + test_expect_code 2 git merge branch2 branch3 >output 2>&1 && + grep "fatal: merge program failed" output && + grep "Should not be doing an octopus" output && + + # Make sure we did not leave stray changes around when no appropriate + # merge strategy was found + git diff --exit-code --name-status && + test_path_is_missing .git/MERGE_HEAD +' + +test_done diff --git a/t/t7609-mergetool--lib.sh b/t/t7609-mergetool--lib.sh index 330d6d603d..8b1c3bd39f 100755 --- a/t/t7609-mergetool--lib.sh +++ b/t/t7609-mergetool--lib.sh @@ -4,6 +4,7 @@ test_description='git mergetool Testing basic merge tools options' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_expect_success 'mergetool --tool=vimdiff creates the expected layout' ' diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 852498ff06..2724a44fe3 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -164,7 +164,6 @@ test_expect_success 'prefetch multiple remotes' ' test_cmp_rev refs/remotes/remote1/one refs/prefetch/remotes/remote1/one && test_cmp_rev refs/remotes/remote2/two refs/prefetch/remotes/remote2/two && - test_cmp_config refs/prefetch/ log.excludedecoration && git log --oneline --decorate --all >log && ! grep "prefetch" log && @@ -175,26 +174,6 @@ test_expect_success 'prefetch multiple remotes' ' test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt ' -test_expect_success 'prefetch and existing log.excludeDecoration values' ' - git config --unset-all log.excludeDecoration && - git config log.excludeDecoration refs/remotes/remote1/ && - git maintenance run --task=prefetch && - - git config --get-all log.excludeDecoration >out && - grep refs/remotes/remote1/ out && - grep refs/prefetch/ out && - - git log --oneline --decorate --all >log && - ! grep "prefetch" log && - ! grep "remote1" log && - grep "remote2" log && - - # a second run does not change the config - git maintenance run --task=prefetch && - git log --oneline --decorate --all >log2 && - test_cmp log log2 -' - test_expect_success 'loose-objects task' ' # Repack everything so we know the state of the object dir git repack -adk && diff --git a/t/t9100-git-svn-basic.sh b/t/t9100-git-svn-basic.sh index 7c5b847f58..fea41b3c36 100755 --- a/t/t9100-git-svn-basic.sh +++ b/t/t9100-git-svn-basic.sh @@ -8,7 +8,6 @@ test_description='git svn basic tests' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME -TEST_FAILS_SANITIZE_LEAK=true . ./lib-git-svn.sh prepare_utf8_locale diff --git a/t/t9122-git-svn-author.sh b/t/t9122-git-svn-author.sh index 527ba3d293..0fc289ae0f 100755 --- a/t/t9122-git-svn-author.sh +++ b/t/t9122-git-svn-author.sh @@ -2,7 +2,6 @@ test_description='git svn authorship' -TEST_FAILS_SANITIZE_LEAK=true . ./lib-git-svn.sh test_expect_success 'setup svn repository' ' diff --git a/t/t9162-git-svn-dcommit-interactive.sh b/t/t9162-git-svn-dcommit-interactive.sh index e2aa8ed88a..b3ce033a0d 100755 --- a/t/t9162-git-svn-dcommit-interactive.sh +++ b/t/t9162-git-svn-dcommit-interactive.sh @@ -4,7 +4,6 @@ test_description='git svn dcommit --interactive series' -TEST_FAILS_SANITIZE_LEAK=true . ./lib-git-svn.sh test_expect_success 'initialize repo' ' diff --git a/t/t9400-git-cvsserver-server.sh b/t/t9400-git-cvsserver-server.sh index 210ddf09e3..379b19f2f8 100755 --- a/t/t9400-git-cvsserver-server.sh +++ b/t/t9400-git-cvsserver-server.sh @@ -221,7 +221,7 @@ test_expect_success 'req_Root (export-all)' \ 'cat request-anonymous | git-cvsserver --export-all pserver "$WORKDIR" >log 2>&1 && sed -ne \$p log | grep "^I LOVE YOU\$"' -test_expect_success 'req_Root failure (export-all w/o whitelist)' \ +test_expect_success 'req_Root failure (export-all w/o directory list)' \ '! (cat request-anonymous | git-cvsserver --export-all pserver >log 2>&1 || false)' test_expect_success 'req_Root (everything together)' \ diff --git a/t/t9700-perl-git.sh b/t/t9700-perl-git.sh index 102c133112..4aa5d90d32 100755 --- a/t/t9700-perl-git.sh +++ b/t/t9700-perl-git.sh @@ -4,17 +4,12 @@ # test_description='perl interface (Git.pm)' -. ./test-lib.sh -if ! test_have_prereq PERL; then - skip_all='skipping perl interface tests, perl not available' - test_done -fi +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-perl.sh -perl -MTest::More -e 0 2>/dev/null || { - skip_all="Perl Test::More unavailable, skipping test" - test_done -} +skip_all_if_no_Test_More # set up test repository @@ -50,11 +45,9 @@ test_expect_success \ git config --add test.pathmulti bar ' -# The external test will outputs its own plan -test_external_has_tap=1 - -test_external_without_stderr \ - 'Perl API' \ - perl "$TEST_DIRECTORY"/t9700/test.pl +test_expect_success 'use t9700/test.pl to test Git.pm' ' + "$PERL_PATH" "$TEST_DIRECTORY"/t9700/test.pl 2>stderr && + test_must_be_empty stderr +' test_done diff --git a/t/t9901-git-web--browse.sh b/t/t9901-git-web--browse.sh index de7152f827..19f56e5680 100755 --- a/t/t9901-git-web--browse.sh +++ b/t/t9901-git-web--browse.sh @@ -5,6 +5,7 @@ test_description='git web--browse basic tests This test checks that git web--browse can handle various valid URLs.' +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh test_web_browse () { diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index 31526e6b64..43de868b80 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -2485,6 +2485,13 @@ test_expect_success 'git config - section' ' EOF ' +test_expect_success 'git config - section include, includeIf' ' + test_completion "git config inclu" <<-\EOF + include.Z + includeIf.Z + EOF +' + test_expect_success 'git config - variable name' ' test_completion "git config log.d" <<-\EOF log.date Z @@ -2493,6 +2500,12 @@ test_expect_success 'git config - variable name' ' EOF ' +test_expect_success 'git config - variable name include' ' + test_completion "git config include.p" <<-\EOF + include.path Z + EOF +' + test_expect_success 'git config - value' ' test_completion "git config color.pager " <<-\EOF false Z diff --git a/t/t9903-bash-prompt.sh b/t/t9903-bash-prompt.sh index 6a30f5719c..d459fae655 100755 --- a/t/t9903-bash-prompt.sh +++ b/t/t9903-bash-prompt.sh @@ -759,4 +759,20 @@ test_expect_success 'prompt - hide if pwd ignored - inside gitdir' ' test_cmp expected "$actual" ' +test_expect_success 'prompt - conflict indicator' ' + printf " (main|CONFLICT)" >expected && + echo "stash" >file && + git stash && + test_when_finished "git stash drop" && + echo "commit" >file && + git commit -m "commit" file && + test_when_finished "git reset --hard HEAD~" && + test_must_fail git stash apply && + ( + GIT_PS1_SHOWCONFLICTSTATE="yes" && + __git_ps1 >"$actual" + ) && + test_cmp expected "$actual" +' + test_done diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 6da7273f1d..c6479f24eb 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -633,7 +633,7 @@ test_hook () { # - Explicitly using test_have_prereq. # # - Implicitly by specifying the prerequisite tag in the calls to -# test_expect_{success,failure} and test_external{,_without_stderr}. +# test_expect_{success,failure} # # The single parameter is the prerequisite tag (a simple word, in all # capital letters by convention). @@ -651,8 +651,7 @@ test_set_prereq () { # test_unset_prereq() !*) ;; - # (Temporary?) whitelist of things we can't easily - # pretend not to support + # List of things we can't easily pretend to not support SYMLINKS) ;; # Inspecting whether GIT_TEST_FAIL_PREREQS is on @@ -836,93 +835,6 @@ test_expect_success () { test_finish_ } -# test_external runs external test scripts that provide continuous -# test output about their progress, and succeeds/fails on -# zero/non-zero exit code. It outputs the test output on stdout even -# in non-verbose mode, and announces the external script with "# run -# <n>: ..." before running it. When providing relative paths, keep in -# mind that all scripts run in "trash directory". -# Usage: test_external description command arguments... -# Example: test_external 'Perl API' perl ../path/to/test.pl -test_external () { - test "$#" = 4 && { test_prereq=$1; shift; } || test_prereq= - test "$#" = 3 || - BUG "not 3 or 4 parameters to test_external" - descr="$1" - shift - test_verify_prereq - export test_prereq - if ! test_skip "$descr" "$@" - then - # Announce the script to reduce confusion about the - # test output that follows. - say_color "" "# run $test_count: $descr ($*)" - # Export TEST_DIRECTORY, TRASH_DIRECTORY and GIT_TEST_LONG - # to be able to use them in script - export TEST_DIRECTORY TRASH_DIRECTORY GIT_TEST_LONG - # Run command; redirect its stderr to &4 as in - # test_run_, but keep its stdout on our stdout even in - # non-verbose mode. - "$@" 2>&4 - if test "$?" = 0 - then - if test $test_external_has_tap -eq 0; then - test_ok_ "$descr" - else - say_color "" "# test_external test $descr was ok" - test_success=$(($test_success + 1)) - fi - else - if test $test_external_has_tap -eq 0; then - test_failure_ "$descr" "$@" - else - say_color error "# test_external test $descr failed: $@" - test_failure=$(($test_failure + 1)) - fi - fi - fi -} - -# Like test_external, but in addition tests that the command generated -# no output on stderr. -test_external_without_stderr () { - # The temporary file has no (and must have no) security - # implications. - tmp=${TMPDIR:-/tmp} - stderr="$tmp/git-external-stderr.$$.tmp" - test_external "$@" 4> "$stderr" - test -f "$stderr" || error "Internal error: $stderr disappeared." - descr="no stderr: $1" - shift - say >&3 "# expecting no stderr from previous command" - if test ! -s "$stderr" - then - rm "$stderr" - - if test $test_external_has_tap -eq 0; then - test_ok_ "$descr" - else - say_color "" "# test_external_without_stderr test $descr was ok" - test_success=$(($test_success + 1)) - fi - else - if test "$verbose" = t - then - output=$(echo; echo "# Stderr is:"; cat "$stderr") - else - output= - fi - # rm first in case test_failure exits. - rm "$stderr" - if test $test_external_has_tap -eq 0; then - test_failure_ "$descr" "$@" "$output" - else - say_color error "# test_external_without_stderr test $descr failed: $@: $output" - test_failure=$(($test_failure + 1)) - fi - fi -} - # debugging-friendly alternatives to "test [-f|-d|-e]" # The commands test the existence or non-existence of $1 test_path_is_file () { diff --git a/t/test-lib.sh b/t/test-lib.sh index 7726d1da88..377cc1c120 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -238,6 +238,9 @@ parse_option () { ;; esac ;; + --invert-exit-code) + invert_exit_code=t + ;; *) echo "error: unknown test option '$opt'" >&2; exit 1 ;; esac @@ -302,6 +305,11 @@ TEST_NUMBER="${TEST_NAME%%-*}" TEST_NUMBER="${TEST_NUMBER#t}" TEST_RESULTS_DIR="$TEST_OUTPUT_DIRECTORY/test-results" TEST_RESULTS_BASE="$TEST_RESULTS_DIR/$TEST_NAME$TEST_STRESS_JOB_SFX" +TEST_RESULTS_SAN_FILE_PFX=trace +TEST_RESULTS_SAN_DIR_SFX=leak +TEST_RESULTS_SAN_FILE= +TEST_RESULTS_SAN_DIR="$TEST_RESULTS_DIR/$TEST_NAME.$TEST_RESULTS_SAN_DIR_SFX" +TEST_RESULTS_SAN_DIR_NR_LEAKS_STARTUP= TRASH_DIRECTORY="trash directory.$TEST_NAME$TEST_STRESS_JOB_SFX" test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY" case "$TRASH_DIRECTORY" in @@ -309,6 +317,16 @@ case "$TRASH_DIRECTORY" in *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;; esac +# Utility functions using $TEST_RESULTS_* variables +nr_san_dir_leaks_ () { + # stderr piped to /dev/null because the directory may have + # been "rmdir"'d already. + find "$TEST_RESULTS_SAN_DIR" \ + -type f \ + -name "$TEST_RESULTS_SAN_FILE_PFX.*" 2>/dev/null | + wc -l +} + # If --stress was passed, run this test repeatedly in several parallel loops. if test "$GIT_TEST_STRESS_STARTED" = "done" then @@ -557,14 +575,19 @@ then : nothing } else + _USE_GLIBC_TUNABLES= + if _GLIBC_VERSION=$(getconf GNU_LIBC_VERSION 2>/dev/null) && + _GLIBC_VERSION=${_GLIBC_VERSION#"glibc "} && + expr 2.34 \<= "$_GLIBC_VERSION" >/dev/null + then + _USE_GLIBC_TUNABLES=YesPlease + fi setup_malloc_check () { local g local t MALLOC_CHECK_=3 MALLOC_PERTURB_=165 export MALLOC_CHECK_ MALLOC_PERTURB_ - if _GLIBC_VERSION=$(getconf GNU_LIBC_VERSION 2>/dev/null) && - _GLIBC_VERSION=${_GLIBC_VERSION#"glibc "} && - expr 2.34 \<= "$_GLIBC_VERSION" >/dev/null + if test -n "$_USE_GLIBC_TUNABLES" then g= LD_PRELOAD="libc_malloc_debug.so.0" @@ -788,15 +811,31 @@ test_ok_ () { finalize_test_case_output ok "$@" } +_invert_exit_code_failure_end_blurb () { + say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code" +} + test_failure_ () { failure_label=$1 test_failure=$(($test_failure + 1)) - say_color error "not ok $test_count - $1" + local pfx="" + if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE" + then + pfx="# TODO induced breakage (--invert-exit-code):" + fi + say_color error "not ok $test_count - ${pfx:+$pfx }$1" shift printf '%s\n' "$*" | sed -e 's/^/# /' if test -n "$immediate" then say_color error "1..$test_count" + if test -n "$invert_exit_code" + then + finalize_test_output + _invert_exit_code_failure_end_blurb + GIT_EXIT_OK=t + exit 0 + fi _error_exit fi finalize_test_case_output failure "$failure_label" "$@" @@ -804,14 +843,14 @@ test_failure_ () { test_known_broken_ok_ () { test_fixed=$(($test_fixed+1)) - say_color error "ok $test_count - $@ # TODO known breakage vanished" - finalize_test_case_output fixed "$@" + say_color error "ok $test_count - $1 # TODO known breakage vanished" + finalize_test_case_output fixed "$1" } test_known_broken_failure_ () { test_broken=$(($test_broken+1)) - say_color warn "not ok $test_count - $@ # TODO known breakage" - finalize_test_case_output broken "$@" + say_color warn "not ok $test_count - $1 # TODO known breakage" + finalize_test_case_output broken "$1" } test_debug () { @@ -1168,9 +1207,67 @@ test_atexit_handler () { teardown_malloc_check } -test_done () { - GIT_EXIT_OK=t +sanitize_leak_log_message_ () { + local new="$1" && + local old="$2" && + local file="$3" && + + printf "With SANITIZE=leak at exit we have %d leak logs, but started with %d + +This means that we have a blindspot where git is leaking but we're +losing the exit code somewhere, or not propagating it appropriately +upwards! + +See the logs at \"%s.*\"; +those logs are reproduced below." \ + "$new" "$old" "$file" +} + +check_test_results_san_file_ () { + if test -z "$TEST_RESULTS_SAN_FILE" + then + return + fi && + local old="$TEST_RESULTS_SAN_DIR_NR_LEAKS_STARTUP" && + local new="$(nr_san_dir_leaks_)" && + + if test $new -le $old + then + return + fi && + local out="$(sanitize_leak_log_message_ "$new" "$old" "$TEST_RESULTS_SAN_FILE")" && + say_color error "$out" && + if test "$old" != 0 + then + echo && + say_color error "The logs include output from past runs to avoid" && + say_color error "that remove 'test-results' between runs." + fi && + say_color error "$(cat "$TEST_RESULTS_SAN_FILE".*)" && + if test -n "$passes_sanitize_leak" && test "$test_failure" = 0 + then + say "As TEST_PASSES_SANITIZE_LEAK=true and our logs show we're leaking, exit non-zero!" && + invert_exit_code=t + elif test -n "$passes_sanitize_leak" + then + say "As TEST_PASSES_SANITIZE_LEAK=true and our logs show we're leaking, and we're failing for other reasons too..." && + invert_exit_code= + elif test -n "$sanitize_leak_check" && test "$test_failure" = 0 + then + say "As TEST_PASSES_SANITIZE_LEAK=true isn't set the above leak is 'ok' with GIT_TEST_PASSING_SANITIZE_LEAK=check" && + invert_exit_code= + elif test -n "$sanitize_leak_check" + then + say "As TEST_PASSES_SANITIZE_LEAK=true isn't set the above leak is 'ok' with GIT_TEST_PASSING_SANITIZE_LEAK=check" && + invert_exit_code=t + else + say "With GIT_TEST_SANITIZE_LEAK_LOG=true our logs revealed a memory leak, exit non-zero!" && + invert_exit_code=t + fi +} + +test_done () { # Run the atexit commands _before_ the trash directory is # removed, so the commands can access pidfiles and socket files. test_atexit_handler @@ -1210,28 +1307,32 @@ test_done () { fi case "$test_failure" in 0) - if test $test_external_has_tap -eq 0 + if test $test_remaining -gt 0 then - if test $test_remaining -gt 0 - then - say_color pass "# passed all $msg" - fi - - # Maybe print SKIP message - test -z "$skip_all" || skip_all="# SKIP $skip_all" - case "$test_count" in - 0) - say "1..$test_count${skip_all:+ $skip_all}" - ;; - *) - test -z "$skip_all" || - say_color warn "$skip_all" - say "1..$test_count" - ;; - esac + say_color pass "# passed all $msg" fi - if test -z "$debug" && test -n "$remove_trash" + # Maybe print SKIP message + test -z "$skip_all" || skip_all="# SKIP $skip_all" + case "$test_count" in + 0) + say "1..$test_count${skip_all:+ $skip_all}" + ;; + *) + test -z "$skip_all" || + say_color warn "$skip_all" + say "1..$test_count" + ;; + esac + + if test -n "$stress" && test -n "$invert_exit_code" + then + # We're about to move our "$TRASH_DIRECTORY" + # to "$TRASH_DIRECTORY.stress-failed" if + # --stress is combined with + # --invert-exit-code. + say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'" + elif test -z "$debug" && test -n "$remove_trash" then test -d "$TRASH_DIRECTORY" || error "Tests passed but trash directory already removed before test cleanup; aborting" @@ -1244,17 +1345,35 @@ test_done () { } || error "Tests passed but test cleanup failed; aborting" fi + + check_test_results_san_file_ "$test_failure" + + if test -z "$skip_all" && test -n "$invert_exit_code" + then + say_color warn "# faking up non-zero exit with --invert-exit-code" + GIT_EXIT_OK=t + exit 1 + fi + test_at_end_hook_ + GIT_EXIT_OK=t exit 0 ;; *) - if test $test_external_has_tap -eq 0 + say_color error "# failed $test_failure among $msg" + say "1..$test_count" + + check_test_results_san_file_ "$test_failure" + + if test -n "$invert_exit_code" then - say_color error "# failed $test_failure among $msg" - say "1..$test_count" + _invert_exit_code_failure_end_blurb + GIT_EXIT_OK=t + exit 0 fi + GIT_EXIT_OK=t exit 1 ;; esac @@ -1387,14 +1506,12 @@ fi GITPERLLIB="$GIT_BUILD_DIR"/perl/build/lib export GITPERLLIB test -d "$GIT_BUILD_DIR"/templates/blt || { - error "You haven't built things yet, have you?" + BAIL_OUT "You haven't built things yet, have you?" } if ! test -x "$GIT_BUILD_DIR"/t/helper/test-tool$X then - echo >&2 'You need to build test-tool:' - echo >&2 'Run "make t/helper/test-tool" in the source (toplevel) directory' - exit 1 + BAIL_OUT 'You need to build test-tool; Run "make t/helper/test-tool" in the source (toplevel) directory' fi # Are we running this test at all? @@ -1408,24 +1525,70 @@ then test_done fi -# skip non-whitelisted tests when compiled with SANITIZE=leak +BAIL_OUT_ENV_NEEDS_SANITIZE_LEAK () { + BAIL_OUT "$1 has no effect except when compiled with SANITIZE=leak" +} + if test -n "$SANITIZE_LEAK" then - if test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false + # Normalize with test_bool_env + passes_sanitize_leak= + + # We need to see TEST_PASSES_SANITIZE_LEAK in "git + # env--helper" (via test_bool_env) + export TEST_PASSES_SANITIZE_LEAK + if test_bool_env TEST_PASSES_SANITIZE_LEAK false + then + passes_sanitize_leak=t + fi + + if test "$GIT_TEST_PASSING_SANITIZE_LEAK" = "check" then - # We need to see it in "git env--helper" (via - # test_bool_env) - export TEST_PASSES_SANITIZE_LEAK + sanitize_leak_check=t + if test -n "$invert_exit_code" + then + BAIL_OUT "cannot use --invert-exit-code under GIT_TEST_PASSING_SANITIZE_LEAK=check" + fi - if ! test_bool_env TEST_PASSES_SANITIZE_LEAK false + if test -z "$passes_sanitize_leak" then - skip_all="skipping $this_test under GIT_TEST_PASSING_SANITIZE_LEAK=true" - test_done + say "in GIT_TEST_PASSING_SANITIZE_LEAK=check mode, setting --invert-exit-code for TEST_PASSES_SANITIZE_LEAK != true" + invert_exit_code=t fi + elif test -z "$passes_sanitize_leak" && + test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false + then + skip_all="skipping $this_test under GIT_TEST_PASSING_SANITIZE_LEAK=true" + test_done fi -elif test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false + + if test_bool_env GIT_TEST_SANITIZE_LEAK_LOG false + then + if ! mkdir -p "$TEST_RESULTS_SAN_DIR" + then + BAIL_OUT "cannot create $TEST_RESULTS_SAN_DIR" + fi && + TEST_RESULTS_SAN_FILE="$TEST_RESULTS_SAN_DIR/$TEST_RESULTS_SAN_FILE_PFX" + + # In case "test-results" is left over from a previous + # run: Only report if new leaks show up. + TEST_RESULTS_SAN_DIR_NR_LEAKS_STARTUP=$(nr_san_dir_leaks_) + + # Don't litter *.leak dirs if there was nothing to report + test_atexit "rmdir \"$TEST_RESULTS_SAN_DIR\" 2>/dev/null || :" + + prepend_var LSAN_OPTIONS : dedup_token_length=9999 + prepend_var LSAN_OPTIONS : log_exe_name=1 + prepend_var LSAN_OPTIONS : log_path=\"$TEST_RESULTS_SAN_FILE\" + export LSAN_OPTIONS + fi +elif test "$GIT_TEST_PASSING_SANITIZE_LEAK" = "check" || + test_bool_env GIT_TEST_PASSING_SANITIZE_LEAK false +then + BAIL_OUT_ENV_NEEDS_SANITIZE_LEAK "GIT_TEST_PASSING_SANITIZE_LEAK=true" +elif test_bool_env GIT_TEST_SANITIZE_LEAK_LOG false then - BAIL_OUT "GIT_TEST_PASSING_SANITIZE_LEAK=true has no effect except when compiled with SANITIZE=leak" + BAIL_OUT_ENV_NEEDS_SANITIZE_LEAK "GIT_TEST_SANITIZE_LEAK_LOG=true" fi # Last-minute variable setup @@ -1448,9 +1611,7 @@ remove_trash_directory () { # Test repository remove_trash_directory "$TRASH_DIRECTORY" || { - GIT_EXIT_OK=t - echo >&5 "FATAL: Cannot prepare test area" - exit 1 + BAIL_OUT 'cannot prepare test area' } remove_trash=t @@ -1466,7 +1627,7 @@ fi # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). -cd -P "$TRASH_DIRECTORY" || exit 1 +cd -P "$TRASH_DIRECTORY" || BAIL_OUT "cannot cd -P to \"$TRASH_DIRECTORY\"" start_test_output "$0" diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index c5c8cfbbaa..37a3163be1 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -479,9 +479,12 @@ static void fn_param_fl(const char *file, int line, const char *param, { const char *event_name = "def_param"; struct json_writer jw = JSON_WRITER_INIT; + enum config_scope scope = current_config_scope(); + const char *scope_name = config_scope_name(scope); jw_object_begin(&jw, 0); event_fmt_prepare(event_name, file, line, NULL, &jw); + jw_object_string(&jw, "scope", scope_name); jw_object_string(&jw, "param", param); jw_object_string(&jw, "value", value); jw_end(&jw); diff --git a/trace2/tr2_tgt_normal.c b/trace2/tr2_tgt_normal.c index c42fbade7f..69f8033077 100644 --- a/trace2/tr2_tgt_normal.c +++ b/trace2/tr2_tgt_normal.c @@ -298,8 +298,11 @@ static void fn_param_fl(const char *file, int line, const char *param, const char *value) { struct strbuf buf_payload = STRBUF_INIT; + enum config_scope scope = current_config_scope(); + const char *scope_name = config_scope_name(scope); - strbuf_addf(&buf_payload, "def_param %s=%s", param, value); + strbuf_addf(&buf_payload, "def_param scope:%s %s=%s", scope_name, param, + value); normal_io_write_fl(file, line, &buf_payload); strbuf_release(&buf_payload); } diff --git a/trace2/tr2_tgt_perf.c b/trace2/tr2_tgt_perf.c index a1eff8bea3..8cb792488c 100644 --- a/trace2/tr2_tgt_perf.c +++ b/trace2/tr2_tgt_perf.c @@ -441,12 +441,17 @@ static void fn_param_fl(const char *file, int line, const char *param, { const char *event_name = "def_param"; struct strbuf buf_payload = STRBUF_INIT; + struct strbuf scope_payload = STRBUF_INIT; + enum config_scope scope = current_config_scope(); + const char *scope_name = config_scope_name(scope); strbuf_addf(&buf_payload, "%s:%s", param, value); + strbuf_addf(&scope_payload, "%s:%s", "scope", scope_name); - perf_io_write_fl(file, line, event_name, NULL, NULL, NULL, NULL, - &buf_payload); + perf_io_write_fl(file, line, event_name, NULL, NULL, NULL, + scope_payload.buf, &buf_payload); strbuf_release(&buf_payload); + strbuf_release(&scope_payload); } static void fn_repo_fl(const char *file, int line, diff --git a/transport.c b/transport.c index 52db7a3cb0..b51e991e44 100644 --- a/transport.c +++ b/transport.c @@ -940,7 +940,7 @@ static int external_specification_len(const char *url) return strchr(url, ':') - url; } -static const struct string_list *protocol_whitelist(void) +static const struct string_list *protocol_allow_list(void) { static int enabled = -1; static struct string_list allowed = STRING_LIST_INIT_DUP; @@ -1020,9 +1020,9 @@ static enum protocol_allow_config get_protocol_config(const char *type) int is_transport_allowed(const char *type, int from_user) { - const struct string_list *whitelist = protocol_whitelist(); - if (whitelist) - return string_list_has_string(whitelist, type); + const struct string_list *allow_list = protocol_allow_list(); + if (allow_list) + return string_list_has_string(allow_list, type); switch (get_protocol_config(type)) { case PROTOCOL_ALLOW_ALWAYS: diff --git a/tree-walk.c b/tree-walk.c index 506234b4b8..74f4d710e8 100644 --- a/tree-walk.c +++ b/tree-walk.c @@ -47,17 +47,20 @@ static int decode_tree_entry(struct tree_desc *desc, const char *buf, unsigned l /* Initialize the descriptor entry */ desc->entry.path = path; - desc->entry.mode = canon_mode(mode); + desc->entry.mode = (desc->flags & TREE_DESC_RAW_MODES) ? mode : canon_mode(mode); desc->entry.pathlen = len - 1; oidread(&desc->entry.oid, (const unsigned char *)path + len); return 0; } -static int init_tree_desc_internal(struct tree_desc *desc, const void *buffer, unsigned long size, struct strbuf *err) +static int init_tree_desc_internal(struct tree_desc *desc, const void *buffer, + unsigned long size, struct strbuf *err, + enum tree_desc_flags flags) { desc->buffer = buffer; desc->size = size; + desc->flags = flags; if (size) return decode_tree_entry(desc, buffer, size, err); return 0; @@ -66,15 +69,16 @@ static int init_tree_desc_internal(struct tree_desc *desc, const void *buffer, u void init_tree_desc(struct tree_desc *desc, const void *buffer, unsigned long size) { struct strbuf err = STRBUF_INIT; - if (init_tree_desc_internal(desc, buffer, size, &err)) + if (init_tree_desc_internal(desc, buffer, size, &err, 0)) die("%s", err.buf); strbuf_release(&err); } -int init_tree_desc_gently(struct tree_desc *desc, const void *buffer, unsigned long size) +int init_tree_desc_gently(struct tree_desc *desc, const void *buffer, unsigned long size, + enum tree_desc_flags flags) { struct strbuf err = STRBUF_INIT; - int result = init_tree_desc_internal(desc, buffer, size, &err); + int result = init_tree_desc_internal(desc, buffer, size, &err, flags); if (result) error("%s", err.buf); strbuf_release(&err); diff --git a/tree-walk.h b/tree-walk.h index a5058469e9..6305d53150 100644 --- a/tree-walk.h +++ b/tree-walk.h @@ -34,6 +34,11 @@ struct tree_desc { /* counts the number of bytes left in the `buffer`. */ unsigned int size; + + /* option flags passed via init_tree_desc_gently() */ + enum tree_desc_flags { + TREE_DESC_RAW_MODES = (1 << 0), + } flags; }; /** @@ -79,7 +84,8 @@ int update_tree_entry_gently(struct tree_desc *); */ void init_tree_desc(struct tree_desc *desc, const void *buf, unsigned long size); -int init_tree_desc_gently(struct tree_desc *desc, const void *buf, unsigned long size); +int init_tree_desc_gently(struct tree_desc *desc, const void *buf, unsigned long size, + enum tree_desc_flags flags); /* * Visit the next entry in a tree. Returns 1 when there are more entries diff --git a/unpack-trees.c b/unpack-trees.c index d561ca01ed..90b92114be 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -487,7 +487,7 @@ static int check_updates(struct unpack_trees_options *o, errs |= run_parallel_checkout(&state, pc_workers, pc_threshold, progress, &cnt); stop_progress(&progress); - errs |= finish_delayed_checkout(&state, NULL, o->verbose_update); + errs |= finish_delayed_checkout(&state, o->verbose_update); git_attr_set_direction(GIT_ATTR_CHECKIN); if (o->clone) @@ -1070,6 +1070,67 @@ static struct cache_entry *create_ce_entry(const struct traverse_info *info, } /* + * Determine whether the path specified by 'p' should be unpacked as a new + * sparse directory in a sparse index. A new sparse directory 'A/': + * - must be outside the sparse cone. + * - must not already be in the index (i.e., no index entry with name 'A/' + * exists). + * - must not have any child entries in the index (i.e., no index entry + * 'A/<something>' exists). + * If 'p' meets the above requirements, return 1; otherwise, return 0. + */ +static int entry_is_new_sparse_dir(const struct traverse_info *info, + const struct name_entry *p) +{ + int res, pos; + struct strbuf dirpath = STRBUF_INIT; + struct unpack_trees_options *o = info->data; + + if (!S_ISDIR(p->mode)) + return 0; + + /* + * If the path is inside the sparse cone, it can't be a sparse directory. + */ + strbuf_add(&dirpath, info->traverse_path, info->pathlen); + strbuf_add(&dirpath, p->path, p->pathlen); + strbuf_addch(&dirpath, '/'); + if (path_in_cone_mode_sparse_checkout(dirpath.buf, o->src_index)) { + res = 0; + goto cleanup; + } + + pos = index_name_pos_sparse(o->src_index, dirpath.buf, dirpath.len); + if (pos >= 0) { + /* Path is already in the index, not a new sparse dir */ + res = 0; + goto cleanup; + } + + /* Where would this sparse dir be inserted into the index? */ + pos = -pos - 1; + if (pos >= o->src_index->cache_nr) { + /* + * Sparse dir would be inserted at the end of the index, so we + * know it has no child entries. + */ + res = 1; + goto cleanup; + } + + /* + * If the dir has child entries in the index, the first would be at the + * position the sparse directory would be inserted. If the entry at this + * position is inside the dir, not a new sparse dir. + */ + res = strncmp(o->src_index->cache[pos]->name, dirpath.buf, dirpath.len); + +cleanup: + strbuf_release(&dirpath); + return res; +} + +/* * Note that traverse_by_cache_tree() duplicates some logic in this function * without actually calling it. If you change the logic here you may need to * check and change there as well. @@ -1078,21 +1139,44 @@ static int unpack_single_entry(int n, unsigned long mask, unsigned long dirmask, struct cache_entry **src, const struct name_entry *names, - const struct traverse_info *info) + const struct traverse_info *info, + int *is_new_sparse_dir) { int i; struct unpack_trees_options *o = info->data; unsigned long conflicts = info->df_conflicts | dirmask; + const struct name_entry *p = names; - if (mask == dirmask && !src[0]) - return 0; + *is_new_sparse_dir = 0; + if (mask == dirmask && !src[0]) { + /* + * If we're not in a sparse index, we can't unpack a directory + * without recursing into it, so we return. + */ + if (!o->src_index->sparse_index) + return 0; + + /* Find first entry with a real name (we could use "mask" too) */ + while (!p->mode) + p++; + + /* + * If the directory is completely missing from the index but + * would otherwise be a sparse directory, we should unpack it. + * If not, we'll return and continue recursively traversing the + * tree. + */ + *is_new_sparse_dir = entry_is_new_sparse_dir(info, p); + if (!*is_new_sparse_dir) + return 0; + } /* - * When we have a sparse directory entry for src[0], - * then this isn't necessarily a directory-file conflict. + * When we are unpacking a sparse directory, then this isn't necessarily + * a directory-file conflict. */ - if (mask == dirmask && src[0] && - S_ISSPARSEDIR(src[0]->ce_mode)) + if (mask == dirmask && + (*is_new_sparse_dir || (src[0] && S_ISSPARSEDIR(src[0]->ce_mode)))) conflicts = 0; /* @@ -1352,7 +1436,7 @@ static int unpack_sparse_callback(int n, unsigned long mask, unsigned long dirma { struct cache_entry *src[MAX_UNPACK_TREES + 1] = { NULL, }; struct unpack_trees_options *o = info->data; - int ret; + int ret, is_new_sparse_dir; assert(o->merge); @@ -1376,7 +1460,7 @@ static int unpack_sparse_callback(int n, unsigned long mask, unsigned long dirma * "index" tree (i.e., names[0]) and adjust 'names', 'n', 'mask', and * 'dirmask' accordingly. */ - ret = unpack_single_entry(n - 1, mask >> 1, dirmask >> 1, src, names + 1, info); + ret = unpack_single_entry(n - 1, mask >> 1, dirmask >> 1, src, names + 1, info, &is_new_sparse_dir); if (src[0]) discard_cache_entry(src[0]); @@ -1394,6 +1478,7 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str struct cache_entry *src[MAX_UNPACK_TREES + 1] = { NULL, }; struct unpack_trees_options *o = info->data; const struct name_entry *p = names; + int is_new_sparse_dir; /* Find first entry with a real name (we could use "mask" too) */ while (!p->mode) @@ -1440,7 +1525,7 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str } } - if (unpack_single_entry(n, mask, dirmask, src, names, info) < 0) + if (unpack_single_entry(n, mask, dirmask, src, names, info, &is_new_sparse_dir)) return -1; if (o->merge && src[0]) { @@ -1478,6 +1563,7 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str } if (!is_sparse_directory_entry(src[0], names, info) && + !is_new_sparse_dir && traverse_trees_recursive(n, dirmask, mask & ~dirmask, names, info) < 0) { return -1; diff --git a/upload-pack.c b/upload-pack.c index 09f48317b0..b217a1f469 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -455,6 +455,7 @@ static void create_pack_file(struct upload_pack_data *pack_data, return; fail: + free(output_state); send_client_data(3, abort_msg, sizeof(abort_msg), pack_data->use_sideband); die("git upload-pack: %s", abort_msg); @@ -161,28 +161,6 @@ void xsetenv(const char *name, const char *value, int overwrite) die_errno(_("could not setenv '%s'"), name ? name : "(null)"); } -/* - * Limit size of IO chunks, because huge chunks only cause pain. OS X - * 64-bit is buggy, returning EINVAL if len >= INT_MAX; and even in - * the absence of bugs, large chunks can result in bad latencies when - * you decide to kill the process. - * - * We pick 8 MiB as our default, but if the platform defines SSIZE_MAX - * that is smaller than that, clip it to SSIZE_MAX, as a call to - * read(2) or write(2) larger than that is allowed to fail. As the last - * resort, we allow a port to pass via CFLAGS e.g. "-DMAX_IO_SIZE=value" - * to override this, if the definition of SSIZE_MAX given by the platform - * is broken. - */ -#ifndef MAX_IO_SIZE -# define MAX_IO_SIZE_DEFAULT (8*1024*1024) -# if defined(SSIZE_MAX) && (SSIZE_MAX < MAX_IO_SIZE_DEFAULT) -# define MAX_IO_SIZE SSIZE_MAX -# else -# define MAX_IO_SIZE MAX_IO_SIZE_DEFAULT -# endif -#endif - /** * xopen() is the same as open(), but it die()s if the open() fails. */ @@ -616,10 +594,16 @@ int git_fsync(int fd, enum fsync_action action) } } +static void log_trace_fsync_if(const char *key, intmax_t value) +{ + if (value) + trace2_data_intmax("fsync", the_repository, key, value); +} + void trace_git_fsync_stats(void) { - trace2_data_intmax("fsync", the_repository, "fsync/writeout-only", count_fsync_writeout_only); - trace2_data_intmax("fsync", the_repository, "fsync/hardware-flush", count_fsync_hardware_flush); + log_trace_fsync_if("fsync/writeout-only", count_fsync_writeout_only); + log_trace_fsync_if("fsync/hardware-flush", count_fsync_hardware_flush); } static int warn_if_unremovable(const char *op, const char *file, int rc) diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index 72e25a9ffa..bb56b23f34 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -120,6 +120,7 @@ typedef struct s_bdiffparam { #define xdl_malloc(x) xmalloc(x) +#define xdl_calloc(n, sz) xcalloc(n, sz) #define xdl_free(ptr) free(ptr) #define xdl_realloc(ptr,x) xrealloc(ptr,x) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index 758410c11a..32652ded2d 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -321,12 +321,12 @@ int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, return -1; if (XDF_DIFF_ALG(xpp->flags) == XDF_PATIENCE_DIFF) { - res = xdl_do_patience_diff(mf1, mf2, xpp, xe); + res = xdl_do_patience_diff(xpp, xe); goto out; } if (XDF_DIFF_ALG(xpp->flags) == XDF_HISTOGRAM_DIFF) { - res = xdl_do_histogram_diff(mf1, mf2, xpp, xe); + res = xdl_do_histogram_diff(xpp, xe); goto out; } @@ -337,7 +337,7 @@ int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, * One is to store the forward path and one to store the backward path. */ ndiags = xe->xdf1.nreff + xe->xdf2.nreff + 3; - if (!(kvd = (long *) xdl_malloc((2 * ndiags + 2) * sizeof(long)))) { + if (!XDL_ALLOC_ARRAY(kvd, 2 * ndiags + 2)) { xdl_free_env(xe); return -1; diff --git a/xdiff/xdiffi.h b/xdiff/xdiffi.h index 8f1c7c8b04..126c9d8ff4 100644 --- a/xdiff/xdiffi.h +++ b/xdiff/xdiffi.h @@ -56,9 +56,7 @@ int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr); void xdl_free_script(xdchange_t *xscr); int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb, xdemitconf_t const *xecfg); -int xdl_do_patience_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, - xdfenv_t *env); -int xdl_do_histogram_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, - xdfenv_t *env); +int xdl_do_patience_diff(xpparam_t const *xpp, xdfenv_t *env); +int xdl_do_histogram_diff(xpparam_t const *xpp, xdfenv_t *env); #endif /* #if !defined(XDIFFI_H) */ diff --git a/xdiff/xhistogram.c b/xdiff/xhistogram.c index 01decffc33..16a8fe2f3f 100644 --- a/xdiff/xhistogram.c +++ b/xdiff/xhistogram.c @@ -251,7 +251,7 @@ static int find_lcs(xpparam_t const *xpp, xdfenv_t *env, int line1, int count1, int line2, int count2) { int b_ptr; - int sz, ret = -1; + int ret = -1; struct histindex index; memset(&index, 0, sizeof(index)); @@ -265,23 +265,16 @@ static int find_lcs(xpparam_t const *xpp, xdfenv_t *env, index.rcha.head = NULL; index.table_bits = xdl_hashbits(count1); - sz = index.records_size = 1 << index.table_bits; - sz *= sizeof(struct record *); - if (!(index.records = (struct record **) xdl_malloc(sz))) + index.records_size = 1 << index.table_bits; + if (!XDL_CALLOC_ARRAY(index.records, index.records_size)) goto cleanup; - memset(index.records, 0, sz); - sz = index.line_map_size = count1; - sz *= sizeof(struct record *); - if (!(index.line_map = (struct record **) xdl_malloc(sz))) + index.line_map_size = count1; + if (!XDL_CALLOC_ARRAY(index.line_map, index.line_map_size)) goto cleanup; - memset(index.line_map, 0, sz); - sz = index.line_map_size; - sz *= sizeof(unsigned int); - if (!(index.next_ptrs = (unsigned int *) xdl_malloc(sz))) + if (!XDL_CALLOC_ARRAY(index.next_ptrs, index.line_map_size)) goto cleanup; - memset(index.next_ptrs, 0, sz); /* lines / 4 + 1 comes from xprepare.c:xdl_prepare_ctx() */ if (xdl_cha_init(&index.rcha, sizeof(struct record), count1 / 4 + 1) < 0) @@ -369,8 +362,7 @@ out: return result; } -int xdl_do_histogram_diff(mmfile_t *file1, mmfile_t *file2, - xpparam_t const *xpp, xdfenv_t *env) +int xdl_do_histogram_diff(xpparam_t const *xpp, xdfenv_t *env) { return histogram_diff(xpp, env, env->xdf1.dstart + 1, env->xdf1.dend - env->xdf1.dstart + 1, diff --git a/xdiff/xmacros.h b/xdiff/xmacros.h index ae4636c247..8487bb396f 100644 --- a/xdiff/xmacros.h +++ b/xdiff/xmacros.h @@ -49,5 +49,23 @@ do { \ ((unsigned long) __p[2]) << 16 | ((unsigned long) __p[3]) << 24; \ } while (0) +/* Allocate an array of nr elements, returns NULL on failure */ +#define XDL_ALLOC_ARRAY(p, nr) \ + ((p) = SIZE_MAX / sizeof(*(p)) >= (size_t)(nr) \ + ? xdl_malloc((nr) * sizeof(*(p))) \ + : NULL) + +/* Allocate an array of nr zeroed out elements, returns NULL on failure */ +#define XDL_CALLOC_ARRAY(p, nr) ((p) = xdl_calloc(nr, sizeof(*(p)))) + +/* + * Ensure array p can accommodate at least nr elements, growing the + * array and updating alloc (which is the number of allocated + * elements) as necessary. Frees p and returns -1 on failure, returns + * 0 on success + */ +#define XDL_ALLOC_GROW(p, nr, alloc) \ + (-!((nr) <= (alloc) || \ + ((p) = xdl_alloc_grow_helper((p), (nr), &(alloc), sizeof(*(p)))))) #endif /* #if !defined(XMACROS_H) */ diff --git a/xdiff/xpatience.c b/xdiff/xpatience.c index 1a21c6a74b..a2d8955537 100644 --- a/xdiff/xpatience.c +++ b/xdiff/xpatience.c @@ -69,7 +69,6 @@ struct hashmap { } *entries, *first, *last; /* were common records found? */ unsigned long has_matches; - mmfile_t *file1, *file2; xdfenv_t *env; xpparam_t const *xpp; }; @@ -139,23 +138,17 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map, * * It is assumed that env has been prepared using xdl_prepare(). */ -static int fill_hashmap(mmfile_t *file1, mmfile_t *file2, - xpparam_t const *xpp, xdfenv_t *env, +static int fill_hashmap(xpparam_t const *xpp, xdfenv_t *env, struct hashmap *result, int line1, int count1, int line2, int count2) { - result->file1 = file1; - result->file2 = file2; result->xpp = xpp; result->env = env; /* We know exactly how large we want the hash map */ result->alloc = count1 * 2; - result->entries = (struct entry *) - xdl_malloc(result->alloc * sizeof(struct entry)); - if (!result->entries) + if (!XDL_CALLOC_ARRAY(result->entries, result->alloc)) return -1; - memset(result->entries, 0, result->alloc * sizeof(struct entry)); /* First, fill with entries from the first file */ while (count1--) @@ -200,7 +193,7 @@ static int binary_search(struct entry **sequence, int longest, */ static int find_longest_common_sequence(struct hashmap *map, struct entry **res) { - struct entry **sequence = xdl_malloc(map->nr * sizeof(struct entry *)); + struct entry **sequence; int longest = 0, i; struct entry *entry; @@ -211,7 +204,7 @@ static int find_longest_common_sequence(struct hashmap *map, struct entry **res) */ int anchor_i = -1; - if (!sequence) + if (!XDL_ALLOC_ARRAY(sequence, map->nr)) return -1; for (entry = map->first; entry; entry = entry->next) { @@ -257,8 +250,7 @@ static int match(struct hashmap *map, int line1, int line2) return record1->ha == record2->ha; } -static int patience_diff(mmfile_t *file1, mmfile_t *file2, - xpparam_t const *xpp, xdfenv_t *env, +static int patience_diff(xpparam_t const *xpp, xdfenv_t *env, int line1, int count1, int line2, int count2); static int walk_common_sequence(struct hashmap *map, struct entry *first, @@ -289,8 +281,7 @@ static int walk_common_sequence(struct hashmap *map, struct entry *first, /* Recurse */ if (next1 > line1 || next2 > line2) { - if (patience_diff(map->file1, map->file2, - map->xpp, map->env, + if (patience_diff(map->xpp, map->env, line1, next1 - line1, line2, next2 - line2)) return -1; @@ -329,8 +320,7 @@ static int fall_back_to_classic_diff(struct hashmap *map, * * This function assumes that env was prepared with xdl_prepare_env(). */ -static int patience_diff(mmfile_t *file1, mmfile_t *file2, - xpparam_t const *xpp, xdfenv_t *env, +static int patience_diff(xpparam_t const *xpp, xdfenv_t *env, int line1, int count1, int line2, int count2) { struct hashmap map; @@ -349,7 +339,7 @@ static int patience_diff(mmfile_t *file1, mmfile_t *file2, } memset(&map, 0, sizeof(map)); - if (fill_hashmap(file1, file2, xpp, env, &map, + if (fill_hashmap(xpp, env, &map, line1, count1, line2, count2)) return -1; @@ -377,9 +367,7 @@ static int patience_diff(mmfile_t *file1, mmfile_t *file2, return result; } -int xdl_do_patience_diff(mmfile_t *file1, mmfile_t *file2, - xpparam_t const *xpp, xdfenv_t *env) +int xdl_do_patience_diff(xpparam_t const *xpp, xdfenv_t *env) { - return patience_diff(file1, file2, xpp, env, - 1, env->xdf1.nrec, 1, env->xdf2.nrec); + return patience_diff(xpp, env, 1, env->xdf1.nrec, 1, env->xdf2.nrec); } diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 105752758f..c84549f6c5 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -78,15 +78,14 @@ static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) { return -1; } - if (!(cf->rchash = (xdlclass_t **) xdl_malloc(cf->hsize * sizeof(xdlclass_t *)))) { + if (!XDL_CALLOC_ARRAY(cf->rchash, cf->hsize)) { xdl_cha_free(&cf->ncha); return -1; } - memset(cf->rchash, 0, cf->hsize * sizeof(xdlclass_t *)); cf->alloc = size; - if (!(cf->rcrecs = (xdlclass_t **) xdl_malloc(cf->alloc * sizeof(xdlclass_t *)))) { + if (!XDL_ALLOC_ARRAY(cf->rcrecs, cf->alloc)) { xdl_free(cf->rchash); xdl_cha_free(&cf->ncha); @@ -112,7 +111,6 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t long hi; char const *line; xdlclass_t *rcrec; - xdlclass_t **rcrecs; line = rec->ptr; hi = (long) XDL_HASHLONG(rec->ha, cf->hbits); @@ -128,14 +126,8 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t return -1; } rcrec->idx = cf->count++; - if (cf->count > cf->alloc) { - cf->alloc *= 2; - if (!(rcrecs = (xdlclass_t **) xdl_realloc(cf->rcrecs, cf->alloc * sizeof(xdlclass_t *)))) { - + if (XDL_ALLOC_GROW(cf->rcrecs, cf->count, cf->alloc)) return -1; - } - cf->rcrecs = rcrecs; - } cf->rcrecs[rcrec->idx] = rcrec; rcrec->line = line; rcrec->size = rec->size; @@ -164,7 +156,7 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ unsigned long hav; char const *blk, *cur, *top, *prev; xrecord_t *crec; - xrecord_t **recs, **rrecs; + xrecord_t **recs; xrecord_t **rhash; unsigned long *ha; char *rchg; @@ -178,26 +170,21 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ if (xdl_cha_init(&xdf->rcha, sizeof(xrecord_t), narec / 4 + 1) < 0) goto abort; - if (!(recs = (xrecord_t **) xdl_malloc(narec * sizeof(xrecord_t *)))) + if (!XDL_ALLOC_ARRAY(recs, narec)) goto abort; hbits = xdl_hashbits((unsigned int) narec); hsize = 1 << hbits; - if (!(rhash = (xrecord_t **) xdl_malloc(hsize * sizeof(xrecord_t *)))) + if (!XDL_CALLOC_ARRAY(rhash, hsize)) goto abort; - memset(rhash, 0, hsize * sizeof(xrecord_t *)); nrec = 0; if ((cur = blk = xdl_mmfile_first(mf, &bsize))) { for (top = blk + bsize; cur < top; ) { prev = cur; hav = xdl_hash_record(&cur, top, xpp->flags); - if (nrec >= narec) { - narec *= 2; - if (!(rrecs = (xrecord_t **) xdl_realloc(recs, narec * sizeof(xrecord_t *)))) - goto abort; - recs = rrecs; - } + if (XDL_ALLOC_GROW(recs, nrec + 1, narec)) + goto abort; if (!(crec = xdl_cha_alloc(&xdf->rcha))) goto abort; crec->ptr = prev; @@ -209,15 +196,14 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ } } - if (!(rchg = (char *) xdl_malloc((nrec + 2) * sizeof(char)))) + if (!XDL_CALLOC_ARRAY(rchg, nrec + 2)) goto abort; - memset(rchg, 0, (nrec + 2) * sizeof(char)); if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) && (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF)) { - if (!(rindex = xdl_malloc((nrec + 1) * sizeof(*rindex)))) + if (!XDL_ALLOC_ARRAY(rindex, nrec + 1)) goto abort; - if (!(ha = xdl_malloc((nrec + 1) * sizeof(*ha)))) + if (!XDL_ALLOC_ARRAY(ha, nrec + 1)) goto abort; } @@ -383,11 +369,8 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd xdlclass_t *rcrec; char *dis, *dis1, *dis2; - if (!(dis = (char *) xdl_malloc(xdf1->nrec + xdf2->nrec + 2))) { - + if (!XDL_CALLOC_ARRAY(dis, xdf1->nrec + xdf2->nrec + 2)) return -1; - } - memset(dis, 0, xdf1->nrec + xdf2->nrec + 2); dis1 = dis; dis2 = dis1 + xdf1->nrec + 1; diff --git a/xdiff/xutils.c b/xdiff/xutils.c index 115b2b1640..9e36f24875 100644 --- a/xdiff/xutils.c +++ b/xdiff/xutils.c @@ -432,3 +432,20 @@ int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp, return 0; } + +void* xdl_alloc_grow_helper(void *p, long nr, long *alloc, size_t size) +{ + void *tmp = NULL; + size_t n = ((LONG_MAX - 16) / 2 >= *alloc) ? 2 * *alloc + 16 : LONG_MAX; + if (nr > n) + n = nr; + if (SIZE_MAX / size >= n) + tmp = xdl_realloc(p, n * size); + if (tmp) { + *alloc = n; + } else { + xdl_free(p); + *alloc = 0; + } + return tmp; +} diff --git a/xdiff/xutils.h b/xdiff/xutils.h index fba7bae03c..fd0bba94e8 100644 --- a/xdiff/xutils.h +++ b/xdiff/xutils.h @@ -42,6 +42,7 @@ int xdl_emit_hunk_hdr(long s1, long c1, long s2, long c2, int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp, int line1, int count1, int line2, int count2); - +/* Do not call this function, use XDL_ALLOC_GROW instead */ +void* xdl_alloc_grow_helper(void* p, long nr, long* alloc, size_t size); #endif /* #if !defined(XUTILS_H) */ |