Skip to content

Git designed for AI agents.

ci: show the tests that passed only on re-run

Federico Elia067450578dc2
diff --git a/.github/scripts/rerun-report.sh b/.github/scripts/rerun-report.sh
new file mode 100755
index 0000000..10489fe
--- /dev/null
+++ b/.github/scripts/rerun-report.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Makes the tests that gotestsum re-ran visible on the run page. gotestsum's --rerun-fails-report
+# writes one line per re-run test, "<package>.<Test>: <runs> runs, <failures> failures"; a parent
+# test and its failing subtest have a line each. A test with more runs than failures failed and then
+# passed, which leaves the job green: it gets a warning annotation. Every line goes to the step
+# summary, and the step output reruns=true tells the workflow to keep the test output. Always exits
+# 0: the re-run policy decides the job's result, not this report.
+#
+# Usage: rerun-report.sh <report> <title>
+set -u
+
+report=$1
+title=$2
+
+[ -s "$report" ] || exit 0
+
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+ echo "reruns=true" >> "$GITHUB_OUTPUT"
+fi
+
+awk '{
+ runs = $2; failures = $4
+ if (runs > failures) {
+ line = $0
+ gsub(/%/, "%25", line)
+ print "::warning title=Flaky test (passed on re-run)::" line
+ }
+}' "$report"
+
+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+ {
+ echo "### Tests re-run by gotestsum: $title"
+ echo
+ echo "| Test | Runs | Failures | Outcome |"
+ echo "| --- | --- | --- | --- |"
+ awk '{
+ name = $1; sub(/:$/, "", name)
+ outcome = ($2 > $4) ? "passed on re-run" : "failed"
+ printf "| `%s` | %s | %s | %s |\n", name, $2, $4, outcome
+ }' "$report"
+ echo
+ echo "The output of every attempt is in the tests.json artifact of this job."
+ } >> "$GITHUB_STEP_SUMMARY"
+fi
+
+exit 0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6940c9a..6ba0e38 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -87,7 +87,26 @@ jobs:
mkdir -p "$RUNNER_TEMP/testpath"
ln -s "$PWD/_build/bin/git-filter-repo" "$RUNNER_TEMP/testpath/git-filter-repo"
export PATH="$RUNNER_TEMP/testpath:$PATH"
- make test-go TEST_PACKAGES="$(echo $pkgs)"
+ make test-go TEST_PACKAGES="$(echo $pkgs)" \
+ TEST_JSON_REPORT="$RUNNER_TEMP/tests.json" TEST_RERUN_REPORT="$RUNNER_TEMP/reruns.txt"
+
+ # gotestsum re-runs failed tests; one that then passes leaves the job
+ # green. List the re-run tests on the run page and keep the output of
+ # every attempt as an artifact.
+ - name: Re-run report
+ id: reruns
+ if: always()
+ run: bash .github/scripts/rerun-report.sh "$RUNNER_TEMP/reruns.txt" "test (${{ matrix.group }})"
+
+ - name: Upload the test output of re-run tests
+ if: always() && steps.reruns.outputs.reruns == 'true'
+ uses: actions/upload-artifact@v4
+ with:
+ name: tests-json-${{ matrix.group }}
+ path: |
+ ${{ runner.temp }}/tests.json
+ ${{ runner.temp }}/reruns.txt
+ retention-days: 14
# The Git, service and storage packages again with the reftable reference
# backend (GITALY_TEST_REF_FORMAT=reftable, set by make test-with-reftable),
@@ -144,7 +163,24 @@ jobs:
mkdir -p "$RUNNER_TEMP/testpath"
ln -s "$PWD/_build/bin/git-filter-repo" "$RUNNER_TEMP/testpath/git-filter-repo"
export PATH="$RUNNER_TEMP/testpath:$PATH"
- make test-with-reftable TEST_PACKAGES="$(echo $pkgs)"
+ make test-with-reftable TEST_PACKAGES="$(echo $pkgs)" \
+ TEST_JSON_REPORT="$RUNNER_TEMP/tests.json" TEST_RERUN_REPORT="$RUNNER_TEMP/reruns.txt"
+
+ # As in the test job: list the re-run tests and keep their output.
+ - name: Re-run report
+ id: reruns
+ if: always()
+ run: bash .github/scripts/rerun-report.sh "$RUNNER_TEMP/reruns.txt" "test (reftable, ${{ matrix.group }})"
+
+ - name: Upload the test output of re-run tests
+ if: always() && steps.reruns.outputs.reruns == 'true'
+ uses: actions/upload-artifact@v4
+ with:
+ name: tests-json-reftable-${{ matrix.group }}
+ path: |
+ ${{ runner.temp }}/tests.json
+ ${{ runner.temp }}/reruns.txt
+ retention-days: 14
# Pull requests: the storage core, configuration, CLI and SDK, and the
# storage end-to-end tests. The full suite runs once the change is on main.
diff --git a/Makefile b/Makefile
index 0b5fc2e..aa32fc9 100644
--- a/Makefile
+++ b/Makefile
@@ -248,6 +248,10 @@ TEST_COVERAGE_DIR ?= ${BUILD_DIR}/cover
TEST_TMP_DIR ?=
## Custom options for gosumtest. These options are different from TEST_OPTIONS
GOTESTSUM_OPTIONS ?=
+## Specify the location where gotestsum reports the tests it re-ran in CI (one line per test: runs
+## and failures). Set it here rather than through GOTESTSUM_OPTIONS on the command line, which
+## would replace the --rerun-fails that CI adds below.
+TEST_RERUN_REPORT ?=
## Directory where Gitaly should write logs to during test execution.
TEST_LOG_DIR ?=
TEST_REPO_DIR := ${BUILD_DIR}/testrepos
@@ -261,6 +265,9 @@ ifdef CI
# We introduced this flag in fb75a2f11 (test: Enable the --rerun-fails flag, 2025-03-13) and specified
# it unconditionally. However, it's better to limit the reruns to CI only.
GOTESTSUM_OPTIONS += --rerun-fails --packages '$(TEST_PACKAGES)'
+ifneq (${TEST_RERUN_REPORT},)
+ GOTESTSUM_OPTIONS += --rerun-fails-report '${TEST_RERUN_REPORT}'
+endif
endif
# Git binaries that are eventually embedded into the Gitaly binary.