diff options
Diffstat (limited to 'ci')
-rwxr-xr-x | ci/check-directional-formatting.bash | 27 | ||||
-rwxr-xr-x | ci/check-unsafe-assertions.sh | 18 | ||||
-rwxr-xr-x | ci/check-whitespace.sh | 101 | ||||
-rw-r--r-- | ci/config/README | 14 | ||||
-rwxr-xr-x | ci/install-dependencies.sh | 168 | ||||
-rwxr-xr-x | ci/install-sdk.ps1 | 12 | ||||
-rwxr-xr-x | ci/lib.sh | 373 | ||||
-rwxr-xr-x | ci/make-test-artifacts.sh | 12 | ||||
-rwxr-xr-x | ci/mount-fileshare.sh | 25 | ||||
-rwxr-xr-x | ci/print-test-failures.sh | 89 | ||||
-rwxr-xr-x | ci/run-build-and-minimal-fuzzers.sh | 31 | ||||
-rwxr-xr-x | ci/run-build-and-tests.sh | 76 | ||||
-rwxr-xr-x | ci/run-static-analysis.sh | 36 | ||||
-rwxr-xr-x | ci/run-style-check.sh | 9 | ||||
-rwxr-xr-x | ci/run-test-slice.sh | 18 | ||||
-rwxr-xr-x | ci/test-documentation.sh | 61 | ||||
-rwxr-xr-x | ci/util/extract-trash-dirs.sh | 50 |
17 files changed, 1120 insertions, 0 deletions
diff --git a/ci/check-directional-formatting.bash b/ci/check-directional-formatting.bash new file mode 100755 index 0000000000..3cbbb7030e --- /dev/null +++ b/ci/check-directional-formatting.bash @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# This script verifies that the non-binary files tracked in the Git index do +# not contain any Unicode directional formatting: such formatting could be used +# to deceive reviewers into interpreting code differently from the compiler. +# This is intended to run on an Ubuntu agent in a GitHub workflow. +# +# To allow translated messages to introduce such directional formatting in the +# future, we exclude the `.po` files from this validation. +# +# Neither GNU grep nor `git grep` (not even with `-P`) handle `\u` as a way to +# specify UTF-8. +# +# To work around that, we use `printf` to produce the pattern as a byte +# sequence, and then feed that to `git grep` as a byte sequence (setting +# `LC_CTYPE` to make sure that the arguments are interpreted as intended). +# +# Note: we need to use Bash here because its `printf` interprets `\uNNNN` as +# UTF-8 code points, as desired. Running this script through Ubuntu's `dash`, +# for example, would use a `printf` that does not understand that syntax. + +# U+202a..U+2a2e: LRE, RLE, PDF, LRO and RLO +# U+2066..U+2069: LRI, RLI, FSI and PDI +regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)' + +! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \ + -- ':(exclude,attr:binary)' ':(exclude)*.po' diff --git a/ci/check-unsafe-assertions.sh b/ci/check-unsafe-assertions.sh new file mode 100755 index 0000000000..233bd9dfbc --- /dev/null +++ b/ci/check-unsafe-assertions.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +make CHECK_ASSERTION_SIDE_EFFECTS=1 >compiler_output 2>compiler_error +if test $? != 0 +then + echo >&2 "ERROR: The compiler could not verify the following assert()" + echo >&2 " calls are free of side-effects. Please replace with" + echo >&2 " ASSERT() calls." + grep undefined.reference.to..not_supposed_to_survive compiler_error | + sed -e s/:[^:]*$// | sort | uniq | tr ':' ' ' | + while read f l + do + printf "${f}:${l}\n " + awk -v start="$l" 'NR >= start { print; if (/\);/) exit }' $f + done + exit 1 +fi +rm compiler_output compiler_error diff --git a/ci/check-whitespace.sh b/ci/check-whitespace.sh new file mode 100755 index 0000000000..c40804394c --- /dev/null +++ b/ci/check-whitespace.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# Check that commits after a specified point do not contain new or modified +# lines with whitespace errors. An optional formatted summary can be generated +# by providing an output file path and url as additional arguments. +# + +baseCommit=$1 +outputFile=$2 +url=$3 + +if test "$#" -ne 1 && test "$#" -ne 3 || test -z "$1" +then + echo "USAGE: $0 <BASE_COMMIT> [<OUTPUT_FILE> <URL>]" + exit 1 +fi + +problems=() +commit= +commitText= +commitTextmd= +goodParent= + +if ! git rev-parse --quiet --verify "${baseCommit}" +then + echo "Invalid <BASE_COMMIT> '${baseCommit}'" + exit 1 +fi + +while read dash sha etc +do + case "${dash}" in + "---") # Line contains commit information. + if test -z "${goodParent}" + then + # Assume the commit has no whitespace errors until detected otherwise. + goodParent=${sha} + fi + + commit="${sha}" + commitText="${sha} ${etc}" + commitTextmd="[${sha}](${url}/commit/${sha}) ${etc}" + ;; + "") + ;; + *) # Line contains whitespace error information for current commit. + if test -n "${goodParent}" + then + problems+=("1) --- ${commitTextmd}") + echo "" + echo "--- ${commitText}" + goodParent= + fi + + case "${dash}" in + *:[1-9]*:) # contains file and line number information + dashend=${dash#*:} + problems+=("[${dash}](${url}/blob/${commit}/${dash%%:*}#L${dashend%:}) ${sha} ${etc}") + ;; + *) + problems+=("\`${dash} ${sha} ${etc}\`") + ;; + esac + echo "${dash} ${sha} ${etc}" + ;; + esac +done <<< "$(git log --check --pretty=format:"---% h% s" "${baseCommit}"..)" + +if test ${#problems[*]} -gt 0 +then + if test -z "${goodParent}" + then + goodParent=${baseCommit: 0:7} + fi + + echo "A whitespace issue was found in one or more of the commits." + echo "Run the following command to resolve whitespace issues:" + echo "git rebase --whitespace=fix ${goodParent}" + + # If target output file is provided, write formatted output. + if test -n "$outputFile" + then + echo "🛑 Please review the Summary output for further information." + ( + echo "### :x: A whitespace issue was found in one or more of the commits." + echo "" + echo "Run these commands to correct the problem:" + echo "1. \`git rebase --whitespace=fix ${goodParent}\`" + echo "1. \`git push --force\`" + echo "" + echo "Errors:" + + for i in "${problems[@]}" + do + echo "${i}" + done + ) >"$outputFile" + fi + + exit 2 +fi diff --git a/ci/config/README b/ci/config/README new file mode 100644 index 0000000000..8de3a04e32 --- /dev/null +++ b/ci/config/README @@ -0,0 +1,14 @@ +You can configure some aspects of the GitHub Actions-based CI on a +per-repository basis by setting "variables" and "secrets" from with the +GitHub web interface. These can be found at: + + https://github.com/<user>/git/settings/secrets/actions + +The following variables can be used: + + - CI_BRANCHES + + By default, CI is run when any branch is pushed. If this variable is + non-empty, then only the branches it lists will run CI. Branch names + should be separated by spaces, and should use their shortened form + (e.g., "main", not "refs/heads/main"). diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh new file mode 100755 index 0000000000..d061a47293 --- /dev/null +++ b/ci/install-dependencies.sh @@ -0,0 +1,168 @@ +#!/bin/sh +# +# Install dependencies required to build and test Git on Linux and macOS +# + +. ${0%/*}/lib.sh + +begin_group "Install dependencies" + +P4WHENCE=https://cdist2.perforce.com/perforce/r23.2 +LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION +JGITWHENCE=https://repo1.maven.org/maven2/org/eclipse/jgit/org.eclipse.jgit.pgm/6.8.0.202311291450-r/org.eclipse.jgit.pgm-6.8.0.202311291450-r.sh + +# Make sudo a no-op and execute the command directly when running as root. +# While using sudo would be fine on most platforms when we are root already, +# some platforms like e.g. Alpine Linux do not have sudo available by default +# and would thus break. +if test "$(id -u)" -eq 0 +then + sudo () { + "$@" + } +fi + +case "$distro" in +alpine-*) + apk add --update shadow sudo meson ninja-build gcc libc-dev curl-dev openssl-dev expat-dev gettext \ + zlib-ng-dev pcre2-dev python3 musl-libintl perl-utils ncurses \ + apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ + bash cvs gnupg perl-cgi perl-dbd-sqlite perl-io-tty >/dev/null + ;; +fedora-*|almalinux-*) + dnf -yq update >/dev/null && + dnf -yq install shadow-utils sudo make gcc findutils diffutils perl python3 gawk gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null + ;; +ubuntu-*|i386/ubuntu-*|debian-*) + # Required so that apt doesn't wait for user input on certain packages. + export DEBIAN_FRONTEND=noninteractive + + case "$distro" in + ubuntu-*) + SVN='libsvn-perl subversion' + LANGUAGES='language-pack-is' + ;; + i386/ubuntu-*) + SVN= + LANGUAGES='language-pack-is' + ;; + *) + SVN='libsvn-perl subversion' + LANGUAGES='locales-all' + ;; + esac + + sudo apt-get -q update + sudo apt-get -q -y install \ + $LANGUAGES apache2 cvs cvsps git gnupg $SVN \ + make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo default-jre \ + tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \ + libemail-valid-perl libio-pty-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \ + libsecret-1-dev libpcre2-dev meson ninja-build pkg-config \ + ${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE + + case "$distro" in + ubuntu-*) + mkdir --parents "$CUSTOM_PATH" + + wget --quiet --directory-prefix="$CUSTOM_PATH" \ + "$P4WHENCE/bin.linux26x86_64/p4d" \ + "$P4WHENCE/bin.linux26x86_64/p4" && + chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" || { + rm -f "$CUSTOM_PATH/p4" + rm -f "$CUSTOM_PATH/p4d" + } + + wget --quiet \ + "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" && + tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ + -C "$CUSTOM_PATH" --strip-components=1 \ + "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" && + rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" || + rm -f "$CUSTOM_PATH/git-lfs" + + wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" && + chmod a+x "$CUSTOM_PATH/jgit" || + rm -f "$CUSTOM_PATH/jgit" + ;; + esac + ;; +macos-*) + export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 + # Uncomment this if you want to run perf tests: + # brew install gnu-time + brew link --force gettext + + mkdir -p "$CUSTOM_PATH" + wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" && + tar -xf helix-core-server.tgz -C "$CUSTOM_PATH" p4 p4d && + sudo xattr -d com.apple.quarantine "$CUSTOM_PATH/p4" "$CUSTOM_PATH/p4d" 2>/dev/null || true + rm helix-core-server.tgz + + case "$jobname" in + osx-meson) + brew install meson ninja pcre2 + ;; + esac + + if test -n "$CC_PACKAGE" + then + BREW_PACKAGE=${CC_PACKAGE/-/@} + brew install "$BREW_PACKAGE" + brew link "$BREW_PACKAGE" + fi + ;; +esac + +case "$jobname" in +ClangFormat) + sudo apt-get -q update + sudo apt-get -q -y install clang-format + ;; +StaticAnalysis) + sudo apt-get -q update + sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ + libexpat-dev gettext make + ;; +sparse) + sudo apt-get -q update -q + sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev \ + libexpat-dev gettext zlib1g-dev sparse + ;; +Documentation) + sudo apt-get -q update + sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make + + test -n "$ALREADY_HAVE_ASCIIDOCTOR" || + sudo gem install --version 1.5.8 asciidoctor + sudo gem install concurrent-ruby + ;; +esac + +if type p4d >/dev/null 2>&1 && type p4 >/dev/null 2>&1 +then + echo "$(tput setaf 6)Perforce Server Version$(tput sgr0)" + p4d -V + echo "$(tput setaf 6)Perforce Client Version$(tput sgr0)" + p4 -V +else + echo >&2 "::warning:: perforce wasn't installed, see above for clues why" +fi + +if type git-lfs >/dev/null 2>&1 +then + echo "$(tput setaf 6)Git-LFS Version$(tput sgr0)" + git-lfs version +else + echo >&2 "::warning:: git-lfs wasn't installed, see above for clues why" +fi + +if type jgit >/dev/null 2>&1 +then + echo "$(tput setaf 6)JGit Version$(tput sgr0)" + jgit version +else + echo >&2 "::warning:: JGit wasn't installed, see above for clues why" +fi + +end_group "Install dependencies" diff --git a/ci/install-sdk.ps1 b/ci/install-sdk.ps1 new file mode 100755 index 0000000000..66f24838a4 --- /dev/null +++ b/ci/install-sdk.ps1 @@ -0,0 +1,12 @@ +param( + [string]$directory='git-sdk', + [string]$url='https://github.com/git-for-windows/git-sdk-64/releases/download/ci-artifacts/git-sdk-x86_64-minimal.zip' +) + +Invoke-WebRequest "$url" -OutFile git-sdk.zip +Expand-Archive -LiteralPath git-sdk.zip -DestinationPath "$directory" +Remove-Item -Path git-sdk.zip + +New-Item -Path .git/info -ItemType Directory -Force +New-Item -Path .git/info/exclude -ItemType File -Force +Add-Content -Path .git/info/exclude -Value "/$directory" diff --git a/ci/lib.sh b/ci/lib.sh new file mode 100755 index 0000000000..f561884d40 --- /dev/null +++ b/ci/lib.sh @@ -0,0 +1,373 @@ +# Library of functions shared by all CI scripts + +if test true = "$GITHUB_ACTIONS" +then + begin_group () { + need_to_end_group=t + echo "::group::$1" >&2 + set -x + } + + end_group () { + test -n "$need_to_end_group" || return 0 + set +x + need_to_end_group= + echo '::endgroup::' >&2 + } +elif test true = "$GITLAB_CI" +then + begin_group () { + need_to_end_group=t + printf '\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K%s\n' \ + "$(date +%s)" "$(echo "$1" | tr ' ' _)" "$1" + trap "end_group '$1'" EXIT + set -x + } + + end_group () { + test -n "$need_to_end_group" || return 0 + set +x + need_to_end_group= + printf '\e[0Ksection_end:%s:%s\r\e[0K\n' \ + "$(date +%s)" "$(echo "$1" | tr ' ' _)" + trap - EXIT + } +else + begin_group () { :; } + end_group () { :; } + + set -x +fi + +group () { + group="$1" + shift + begin_group "$group" + + # work around `dash` not supporting `set -o pipefail` + ( + "$@" 2>&1 + echo $? >exit.status + ) | + sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' + res=$(cat exit.status) + rm exit.status + + end_group "$group" + return $res +} + +begin_group "CI setup via $(basename $0)" + +# Set 'exit on error' for all CI scripts to let the caller know that +# something went wrong. +# +# We already enabled tracing executed commands earlier. This helps by showing +# how # environment variables are set and dependencies are installed. +set -e + +skip_branch_tip_with_tag () { + # Sometimes, a branch is pushed at the same time the tag that points + # at the same commit as the tip of the branch is pushed, and building + # both at the same time is a waste. + # + # When the build is triggered by a push to a tag, $CI_BRANCH will + # have that tagname, e.g. v2.14.0. Let's see if $CI_BRANCH is + # exactly at a tag, and if so, if it is different from $CI_BRANCH. + # That way, we can tell if we are building the tip of a branch that + # is tagged and we can skip the build because we won't be skipping a + # build of a tag. + + if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) && + test "$TAG" != "$CI_BRANCH" + then + echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)" + exit 0 + fi +} + +# Check whether we can use the path passed via the first argument as Git +# repository. +is_usable_git_repository () { + # We require Git in our PATH, otherwise we cannot access repositories + # at all. + if ! command -v git >/dev/null + then + return 1 + fi + + # And the target directory needs to be a proper Git repository. + if ! git -C "$1" rev-parse 2>/dev/null + then + return 1 + fi +} + +# Save some info about the current commit's tree, so we can skip the build +# job if we encounter the same tree again and can provide a useful info +# message. +save_good_tree () { + if ! is_usable_git_repository . + then + return + fi + + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" + # limit the file size + tail -1000 "$good_trees_file" >"$good_trees_file".tmp + mv "$good_trees_file".tmp "$good_trees_file" +} + +# Skip the build job if the same tree has already been built and tested +# successfully before (e.g. because the branch got rebased, changing only +# the commit messages). +skip_good_tree () { + if test true = "$GITHUB_ACTIONS" + then + return + fi + + if ! is_usable_git_repository . + then + return + fi + + if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" + then + # Haven't seen this tree yet, or no cached good trees file yet. + # Continue the build job. + return + fi + + echo "$good_tree_info" | { + read tree prev_good_commit prev_good_job_number prev_good_job_id + + if test "$CI_JOB_ID" = "$prev_good_job_id" + then + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) + This commit has already been built and tested successfully by this build job. + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + else + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) + This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. + The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + fi + } + + exit 0 +} + +check_unignored_build_artifacts () { + if ! is_usable_git_repository . + then + return + fi + + ! git ls-files --other --exclude-standard --error-unmatch \ + -- ':/*' 2>/dev/null || + { + echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)" + false + } +} + +handle_failed_tests () { + return 1 +} + +create_failed_test_artifacts () { + mkdir -p "${TEST_OUTPUT_DIRECTORY:-t}"/failed-test-artifacts + + for test_exit in "${TEST_OUTPUT_DIRECTORY:-t}"/test-results/*.exit + do + test 0 != "$(cat "$test_exit")" || continue + + test_name="${test_exit%.exit}" + test_name="${test_name##*/}" + printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + echo "The full logs are in the 'print test failures' step below." + echo "See also the 'failed-tests-*' artifacts attached to this run." + cat "${TEST_OUTPUT_DIRECTORY:-t}/test-results/$test_name.markup" + + trash_dir="${TEST_OUTPUT_DIRECTORY:-t}/trash directory.$test_name" + cp "${TEST_OUTPUT_DIRECTORY:-t}/test-results/$test_name.out" "${TEST_OUTPUT_DIRECTORY:-t}"/failed-test-artifacts/ + tar czf "${TEST_OUTPUT_DIRECTORY:-t}/failed-test-artifacts/$test_name.trash.tar.gz" "$trash_dir" + done +} + +# GitHub Action doesn't set TERM, which is required by tput +export TERM=${TERM:-dumb} + +# Clear MAKEFLAGS that may come from the outside world. +export MAKEFLAGS= + +if test true = "$GITHUB_ACTIONS" +then + CI_TYPE=github-actions + CI_BRANCH="$GITHUB_REF" + CI_COMMIT="$GITHUB_SHA" + CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)" + test macos != "$CI_OS_NAME" || CI_OS_NAME=osx + CI_REPO_SLUG="$GITHUB_REPOSITORY" + CI_JOB_ID="$GITHUB_RUN_ID" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + handle_failed_tests () { + echo "FAILED_TEST_ARTIFACTS=${TEST_OUTPUT_DIRECTORY:-t}/failed-test-artifacts" >>$GITHUB_ENV + create_failed_test_artifacts + return 1 + } + + cache_dir="$HOME/none" + + GIT_TEST_OPTS="--github-workflow-markup" + JOBS=10 + + distro=$(echo "$CI_JOB_IMAGE" | tr : -) +elif test true = "$GITLAB_CI" +then + CI_TYPE=gitlab-ci + CI_BRANCH="$CI_COMMIT_REF_NAME" + CI_COMMIT="$CI_COMMIT_SHA" + + case "$OS,$CI_JOB_IMAGE" in + Windows_NT,*) + CI_OS_NAME=windows + JOBS=$NUMBER_OF_PROCESSORS + ;; + *,macos-*) + # GitLab CI has Python installed via multiple package managers, + # most notably via asdf and Homebrew. Ensure that our builds + # pick up the Homebrew one by prepending it to our PATH as the + # asdf one breaks tests. + export PATH="$(brew --prefix)/bin:$PATH" + + CI_OS_NAME=osx + JOBS=$(nproc) + ;; + *,alpine:*|*,fedora:*|*,ubuntu:*|*,i386/ubuntu:*) + CI_OS_NAME=linux + JOBS=$(nproc) + ;; + *) + echo "Could not identify OS image" >&2 + env >&2 + exit 1 + ;; + esac + CI_REPO_SLUG="$CI_PROJECT_PATH" + CI_JOB_ID="$CI_JOB_ID" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + + handle_failed_tests () { + create_failed_test_artifacts + return 1 + } + + cache_dir="$HOME/none" + + distro=$(echo "$CI_JOB_IMAGE" | tr : -) +else + echo "Could not identify CI type" >&2 + env >&2 + exit 1 +fi + +MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS" +GIT_PROVE_OPTS="--timer --jobs $JOBS" + +GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x" +case "$CI_OS_NAME" in +windows|windows_nt) + GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers" + ;; +esac + +export GIT_TEST_OPTS +export GIT_PROVE_OPTS + +good_trees_file="$cache_dir/good-trees" + +mkdir -p "$cache_dir" + +test -n "${DONT_SKIP_TAGS-}" || +skip_branch_tip_with_tag +skip_good_tree + +if test -z "$jobname" +then + jobname="$CI_OS_NAME-$CC" +fi + +export DEVELOPER=1 +export DEFAULT_TEST_TARGET=prove +export GIT_TEST_CLONE_2GB=true +export SKIP_DASHED_BUILT_INS=YesPlease + +case "$distro" in +ubuntu-*) + # Python 2 is end of life, and Ubuntu 23.04 and newer don't actually + # have it anymore. We thus only test with Python 2 on older LTS + # releases. + if test "$distro" = "ubuntu-20.04" + then + PYTHON_PACKAGE=python2 + else + PYTHON_PACKAGE=python3 + fi + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE" + + export GIT_TEST_HTTPD=true + + # The Linux build installs the defined dependency versions below. + # The OS X build installs much more recent versions, whichever + # were recorded in the Homebrew database upon creating the OS X + # image. + # Keep that in mind when you encounter a broken OS X build! + export LINUX_GIT_LFS_VERSION="1.5.2" + ;; +macos-*) + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" + if [ "$jobname" != osx-gcc ] + then + MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes" + fi + ;; +esac + +CUSTOM_PATH="${CUSTOM_PATH:-$HOME/path}" +export PATH="$CUSTOM_PATH:$PATH" + +case "$jobname" in +linux32) + CC=gcc + ;; +linux-meson) + MESONFLAGS="$MESONFLAGS -Dcredential_helpers=libsecret,netrc" + ;; +linux-musl-meson) + MESONFLAGS="$MESONFLAGS -Dtest_utf8_locale=C.UTF-8" + ;; +linux-leaks|linux-reftable-leaks) + export SANITIZE=leak + ;; +linux-asan-ubsan) + export SANITIZE=address,undefined + export NO_SVN_TESTS=LetsSaveSomeTime + MAKEFLAGS="$MAKEFLAGS NO_PYTHON=YepBecauseP4FlakesTooOften" + ;; +osx-meson) + MESONFLAGS="$MESONFLAGS -Dcredential_helpers=osxkeychain" + ;; +esac + +MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" + +end_group "CI setup via $(basename $0)" +set -x diff --git a/ci/make-test-artifacts.sh b/ci/make-test-artifacts.sh new file mode 100755 index 0000000000..74141af0cc --- /dev/null +++ b/ci/make-test-artifacts.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# +# Build Git and store artifacts for testing +# + +mkdir -p "$1" # in case ci/lib.sh decides to quit early + +. ${0%/*}/lib.sh + +group Build make artifacts-tar ARTIFACTS_DIRECTORY="$1" + +check_unignored_build_artifacts diff --git a/ci/mount-fileshare.sh b/ci/mount-fileshare.sh new file mode 100755 index 0000000000..26b58a8096 --- /dev/null +++ b/ci/mount-fileshare.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +die () { + echo "$*" >&2 + exit 1 +} + +test $# = 4 || +die "Usage: $0 <share> <username> <password> <mountpoint>" + +mkdir -p "$4" || die "Could not create $4" + +case "$(uname -s)" in +Linux) + sudo mount -t cifs -o vers=3.0,username="$2",password="$3",dir_mode=0777,file_mode=0777,serverino "$1" "$4" + ;; +Darwin) + pass="$(echo "$3" | sed -e 's/\//%2F/g' -e 's/+/%2B/g')" && + mount -t smbfs,soft "smb://$2:$pass@${1#//}" "$4" + ;; +*) + die "No support for $(uname -s)" + ;; +esac || +die "Could not mount $4" diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh new file mode 100755 index 0000000000..5545e77c13 --- /dev/null +++ b/ci/print-test-failures.sh @@ -0,0 +1,89 @@ +#!/bin/sh +# +# Print output of failing tests +# + +. ${0%/*}/lib.sh + +# Tracing executed commands would produce too much noise in the loop below. +set +x + +cd "${TEST_OUTPUT_DIRECTORY:-t/}" + +if ! ls test-results/*.exit >/dev/null 2>/dev/null +then + echo "Build job failed before the tests could have been run" + exit +fi + +case "$jobname" in +osx-clang|osx-gcc) + # base64 in OSX doesn't wrap its output at 76 columns by + # default, but prints a single, very long line. + base64_opts="-b 76" + ;; +esac + +combined_trash_size=0 +for TEST_EXIT in test-results/*.exit +do + if [ "$(cat "$TEST_EXIT")" != "0" ] + then + TEST_OUT="${TEST_EXIT%exit}out" + echo "------------------------------------------------------------------------" + echo "$(tput setaf 1)${TEST_OUT}...$(tput sgr0)" + echo "------------------------------------------------------------------------" + cat "${TEST_OUT}" + + test_name="${TEST_EXIT%.exit}" + test_name="${test_name##*/}" + trash_dir="trash directory.$test_name" + case "$CI_TYPE" in + github-actions) + mkdir -p failed-test-artifacts + echo "FAILED_TEST_ARTIFACTS=${TEST_OUTPUT_DIRECTORY:-t}/failed-test-artifacts" >>$GITHUB_ENV + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; + gitlab-ci) + mkdir -p failed-test-artifacts + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; + *) + echo "Unhandled CI type: $CI_TYPE" >&2 + exit 1 + ;; + esac + trash_tgz_b64="trash.$test_name.base64" + if [ -d "$trash_dir" ] + then + tar czp "$trash_dir" |base64 $base64_opts >"$trash_tgz_b64" + + trash_size=$(wc -c <"$trash_tgz_b64") + if [ $trash_size -gt 1048576 ] + then + # larger than 1MB + echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, it's too big$(tput sgr0)" + continue + fi + + new_combined_trash_size=$(($combined_trash_size + $trash_size)) + if [ $new_combined_trash_size -gt 1048576 ] + then + echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, there is plenty of trash in there already.$(tput sgr0)" + continue + fi + combined_trash_size=$new_combined_trash_size + + # DO NOT modify these two 'echo'-ed strings below + # without updating 'ci/util/extract-trash-dirs.sh' + # as well. + echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)" + cat "$trash_tgz_b64" + echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)" + fi + fi +done diff --git a/ci/run-build-and-minimal-fuzzers.sh b/ci/run-build-and-minimal-fuzzers.sh new file mode 100755 index 0000000000..e7b97952e7 --- /dev/null +++ b/ci/run-build-and-minimal-fuzzers.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Build and test Git's fuzzers +# + +. ${0%/*}/lib.sh + +group "Build fuzzers" make \ + NO_CURL=NoThanks \ + CC=clang \ + FUZZ_CXX=clang++ \ + CFLAGS="-fsanitize=fuzzer-no-link,address" \ + LIB_FUZZING_ENGINE="-fsanitize=fuzzer,address" \ + fuzz-all + +fuzzers=" +commit-graph +config +credential-from-url-gently +date +pack-headers +pack-idx +parse-attr-line +url-decode-mem +" + +for fuzzer in $fuzzers; do + begin_group "fuzz-$fuzzer" + ./oss-fuzz/fuzz-$fuzzer -verbosity=0 -runs=1 || exit 1 + end_group "fuzz-$fuzzer" +done diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh new file mode 100755 index 0000000000..01823fd0f1 --- /dev/null +++ b/ci/run-build-and-tests.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# +# Build and test Git +# + +. ${0%/*}/lib.sh + +run_tests=t + +case "$jobname" in +linux-breaking-changes) + export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main + export WITH_BREAKING_CHANGES=YesPlease + ;; +linux-TEST-vars) + export OPENSSL_SHA1_UNSAFE=YesPlease + export GIT_TEST_SPLIT_INDEX=yes + export GIT_TEST_FULL_IN_PACK_ARRAY=true + export GIT_TEST_OE_SIZE=10 + export GIT_TEST_OE_DELTA_SIZE=5 + export GIT_TEST_COMMIT_GRAPH=1 + export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1 + export GIT_TEST_MULTI_PACK_INDEX=1 + export GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=1 + export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master + export GIT_TEST_NO_WRITE_REV_INDEX=1 + export GIT_TEST_CHECKOUT_WORKERS=2 + export GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=1 + ;; +linux-clang) + export GIT_TEST_DEFAULT_HASH=sha1 + ;; +linux-sha256) + export GIT_TEST_DEFAULT_HASH=sha256 + ;; +linux-reftable|linux-reftable-leaks|osx-reftable) + export GIT_TEST_DEFAULT_REF_FORMAT=reftable + ;; +pedantic) + # Don't run the tests; we only care about whether Git can be + # built. + export DEVOPTS=pedantic + run_tests= + ;; +esac + +case "$jobname" in +*-meson) + group "Configure" meson setup build . \ + --fatal-meson-warnings \ + --warnlevel 2 --werror \ + --wrap-mode nofallback \ + -Dfuzzers=true \ + -Dtest_output_directory="${TEST_OUTPUT_DIRECTORY:-$(pwd)/t}" \ + $MESONFLAGS + group "Build" meson compile -C build -- + if test -n "$run_tests" + then + group "Run tests" meson test -C build --print-errorlogs --test-args="$GIT_TEST_OPTS" || ( + ./t/aggregate-results.sh "${TEST_OUTPUT_DIRECTORY:-t}/test-results" + handle_failed_tests + ) + fi + ;; +*) + group Build make + if test -n "$run_tests" + then + group "Run tests" make test || + handle_failed_tests + fi + ;; +esac + +check_unignored_build_artifacts +save_good_tree diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh new file mode 100755 index 0000000000..9e9c72681d --- /dev/null +++ b/ci/run-static-analysis.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Perform various static code analysis checks +# + +. ${0%/*}/lib.sh + +make coccicheck + +set +x + +fail= +for cocci_patch in contrib/coccinelle/*.patch +do + if test -s "$cocci_patch" + then + echo "$(tput setaf 1)Coccinelle suggests the following changes in '$cocci_patch':$(tput sgr0)" + cat "$cocci_patch" + fail=UnfortunatelyYes + fi +done + +if test -n "$fail" +then + echo "$(tput setaf 1)error: Coccinelle suggested some changes$(tput sgr0)" + exit 1 +fi + +make check-headers || +exit 1 + +make check-pot + +${0%/*}/check-unsafe-assertions.sh + +save_good_tree diff --git a/ci/run-style-check.sh b/ci/run-style-check.sh new file mode 100755 index 0000000000..0832c19df0 --- /dev/null +++ b/ci/run-style-check.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Perform style check +# + +baseCommit=$1 + +git clang-format --style=file:.clang-format \ + --diff --extensions c,h "$baseCommit" diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh new file mode 100755 index 0000000000..0444c79c02 --- /dev/null +++ b/ci/run-test-slice.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# +# Test Git in parallel +# + +. ${0%/*}/lib.sh + +group "Run tests" make --quiet -C t T="$(cd t && + ./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh | + tr '\n' ' ')" || +handle_failed_tests + +# We only have one unit test at the moment, so run it in the first slice +if [ "$1" == "0" ] ; then + group "Run unit tests" make --quiet -C t unit-tests-test-tool +fi + +check_unignored_build_artifacts diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh new file mode 100755 index 0000000000..49f87f50fd --- /dev/null +++ b/ci/test-documentation.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Perform sanity checks on documentation and build it. +# + +. ${0%/*}/lib.sh + +filter_log () { + sed -e '/^GIT_VERSION=/d' \ + -e "/constant Gem::ConfigMap is deprecated/d" \ + -e '/^ \* new asciidoc flags$/d' \ + -e '/stripped namespace before processing/d' \ + -e '/Attributed.*IDs for element/d' \ + -e '/SyntaxWarning: invalid escape sequence/d' \ + "$1" +} + +check_docs () { + test -s "$1"/Documentation/git.html && + test -s "$1"/Documentation/git.xml && + test -s "$1"/Documentation/git.1 && + grep "<meta name=\"generator\" content=\"$2 " "$1"/Documentation/git.html +} + +make check-builtins +make check-docs + +# Build docs with AsciiDoc +make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2) +cat stderr.raw +filter_log stderr.raw >stderr.log +test ! -s stderr.log +check_docs . AsciiDoc + +rm -f stdout.log stderr.log stderr.raw +check_unignored_build_artifacts + +# Build docs with AsciiDoctor +make clean +make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2) +cat stderr.raw +filter_log stderr.raw >stderr.log +test ! -s stderr.log +check_docs . Asciidoctor + +rm -f stdout.log stderr.log stderr.raw +check_unignored_build_artifacts + +# Build docs with Meson and AsciiDoc +meson setup build-asciidoc -Ddocs=html,man -Ddocs_backend=asciidoc +meson compile -C build-asciidoc +check_docs build-asciidoc AsciiDoc +rm -rf build-asciidoc + +# Build docs with Meson and AsciiDoctor +meson setup build-asciidoctor -Ddocs=html,man -Ddocs_backend=asciidoctor +meson compile -C build-asciidoctor +check_docs build-asciidoctor Asciidoctor +rm -rf build-asciidoctor + +save_good_tree diff --git a/ci/util/extract-trash-dirs.sh b/ci/util/extract-trash-dirs.sh new file mode 100755 index 0000000000..8e67bec21a --- /dev/null +++ b/ci/util/extract-trash-dirs.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +error () { + echo >&2 "error: $@" + exit 1 +} + +find_embedded_trash () { + while read -r line + do + case "$line" in + *Start\ of\ trash\ directory\ of\ \'t[0-9][0-9][0-9][0-9]-*\':*) + test_name="${line#*\'}" + test_name="${test_name%\'*}" + + return 0 + esac + done + + return 1 +} + +extract_embedded_trash () { + while read -r line + do + case "$line" in + *End\ of\ trash\ directory\ of\ \'$test_name\'*) + return + ;; + *) + printf '%s\n' "$line" + ;; + esac + done + + error "unexpected end of input" +} + +# Raw logs from Linux build jobs have CRLF line endings, while OSX +# build jobs mostly have CRCRLF, except an odd line every now and +# then that has CRCRCRLF. 'base64 -d' from 'coreutils' doesn't like +# CRs and complains about "invalid input", so remove all CRs at the +# end of lines. +sed -e 's/\r*$//' | \ +while find_embedded_trash +do + echo "Extracting trash directory of '$test_name'" + + extract_embedded_trash |base64 -d |tar xzp +done |