Git designed for AI agents.
ci: show the tests that passed only on re-run
Federico Elia067450578dc2
diff --git a/.github/scripts/rerun-report.sh b/.github/scripts/rerun-report.shnew file mode 100755index 0000000..10489fe--- /dev/null+++ b/.github/scripts/rerun-report.sh@@ -0,0 +1,46 @@+#!/usr/bin/env bash+# Makes the tests that gotestsum re-ran visible on the run page. gotestsum's --rerun-fails-report+# writes one line per re-run test, "<package>.<Test>: <runs> runs, <failures> failures"; a parent+# test and its failing subtest have a line each. A test with more runs than failures failed and then+# passed, which leaves the job green: it gets a warning annotation. Every line goes to the step+# summary, and the step output reruns=true tells the workflow to keep the test output. Always exits+# 0: the re-run policy decides the job's result, not this report.+#+# Usage: rerun-report.sh <report> <title>+set -u++report=$1+title=$2++[ -s "$report" ] || exit 0++if [ -n "${GITHUB_OUTPUT:-}" ]; then+ echo "reruns=true" >> "$GITHUB_OUTPUT"+fi++awk '{+ runs = $2; failures = $4+ if (runs > failures) {+ line = $0+ gsub(/%/, "%25", line)+ print "::warning title=Flaky test (passed on re-run)::" line+ }+}' "$report"++if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then+ {+ echo "### Tests re-run by gotestsum: $title"+ echo+ echo "| Test | Runs | Failures | Outcome |"+ echo "| --- | --- | --- | --- |"+ awk '{+ name = $1; sub(/:$/, "", name)+ outcome = ($2 > $4) ? "passed on re-run" : "failed"+ printf "| `%s` | %s | %s | %s |\n", name, $2, $4, outcome+ }' "$report"+ echo+ echo "The output of every attempt is in the tests.json artifact of this job."+ } >> "$GITHUB_STEP_SUMMARY"+fi++exit 0diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 6940c9a..6ba0e38 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -87,7 +87,26 @@ jobs:mkdir -p "$RUNNER_TEMP/testpath"ln -s "$PWD/_build/bin/git-filter-repo" "$RUNNER_TEMP/testpath/git-filter-repo"export PATH="$RUNNER_TEMP/testpath:$PATH"- make test-go TEST_PACKAGES="$(echo $pkgs)"+ make test-go TEST_PACKAGES="$(echo $pkgs)" \+ TEST_JSON_REPORT="$RUNNER_TEMP/tests.json" TEST_RERUN_REPORT="$RUNNER_TEMP/reruns.txt"++ # gotestsum re-runs failed tests; one that then passes leaves the job+ # green. List the re-run tests on the run page and keep the output of+ # every attempt as an artifact.+ - name: Re-run report+ id: reruns+ if: always()+ run: bash .github/scripts/rerun-report.sh "$RUNNER_TEMP/reruns.txt" "test (${{ matrix.group }})"++ - name: Upload the test output of re-run tests+ if: always() && steps.reruns.outputs.reruns == 'true'+ uses: actions/upload-artifact@v4+ with:+ name: tests-json-${{ matrix.group }}+ path: |+ ${{ runner.temp }}/tests.json+ ${{ runner.temp }}/reruns.txt+ retention-days: 14# The Git, service and storage packages again with the reftable reference# backend (GITALY_TEST_REF_FORMAT=reftable, set by make test-with-reftable),@@ -144,7 +163,24 @@ jobs:mkdir -p "$RUNNER_TEMP/testpath"ln -s "$PWD/_build/bin/git-filter-repo" "$RUNNER_TEMP/testpath/git-filter-repo"export PATH="$RUNNER_TEMP/testpath:$PATH"- make test-with-reftable TEST_PACKAGES="$(echo $pkgs)"+ make test-with-reftable TEST_PACKAGES="$(echo $pkgs)" \+ TEST_JSON_REPORT="$RUNNER_TEMP/tests.json" TEST_RERUN_REPORT="$RUNNER_TEMP/reruns.txt"++ # As in the test job: list the re-run tests and keep their output.+ - name: Re-run report+ id: reruns+ if: always()+ run: bash .github/scripts/rerun-report.sh "$RUNNER_TEMP/reruns.txt" "test (reftable, ${{ matrix.group }})"++ - name: Upload the test output of re-run tests+ if: always() && steps.reruns.outputs.reruns == 'true'+ uses: actions/upload-artifact@v4+ with:+ name: tests-json-reftable-${{ matrix.group }}+ path: |+ ${{ runner.temp }}/tests.json+ ${{ runner.temp }}/reruns.txt+ retention-days: 14# Pull requests: the storage core, configuration, CLI and SDK, and the# storage end-to-end tests. The full suite runs once the change is on main.diff --git a/Makefile b/Makefileindex 0b5fc2e..aa32fc9 100644--- a/Makefile+++ b/Makefile@@ -248,6 +248,10 @@ TEST_COVERAGE_DIR ?= ${BUILD_DIR}/coverTEST_TMP_DIR ?=## Custom options for gosumtest. These options are different from TEST_OPTIONSGOTESTSUM_OPTIONS ?=+## Specify the location where gotestsum reports the tests it re-ran in CI (one line per test: runs+## and failures). Set it here rather than through GOTESTSUM_OPTIONS on the command line, which+## would replace the --rerun-fails that CI adds below.+TEST_RERUN_REPORT ?=## Directory where Gitaly should write logs to during test execution.TEST_LOG_DIR ?=TEST_REPO_DIR := ${BUILD_DIR}/testrepos@@ -261,6 +265,9 @@ ifdef CI# We introduced this flag in fb75a2f11 (test: Enable the --rerun-fails flag, 2025-03-13) and specified# it unconditionally. However, it's better to limit the reruns to CI only.GOTESTSUM_OPTIONS += --rerun-fails --packages '$(TEST_PACKAGES)'+ifneq (${TEST_RERUN_REPORT},)+ GOTESTSUM_OPTIONS += --rerun-fails-report '${TEST_RERUN_REPORT}'+endifendif# Git binaries that are eventually embedded into the Gitaly binary.