Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

ci: retry failed test steps once in the merge queue (#18287)

Der Gerätd22b43062641
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 9cd169438b..8733c1e43f 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -199,7 +199,7 @@ jobs:
- name: Lint Rust targets
run: cargo clippy --locked --all-targets -- -D warnings
- name: Run Rust tests
- run: cargo test --locked
+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" cargo test --locked
- name: Build runtime image
run: docker build --pull --target runtime -t langfuse-ai-gateway:test .
- name: Smoke test runtime image
@@ -268,6 +268,8 @@ jobs:
run: node --test scripts/agents/sync-agent-shims.test.mjs
- name: cursor PR label tests
run: node --test scripts/github/label-cursor-prs.test.mjs
+ - name: merge-queue retry helper tests
+ run: bash scripts/ci/retry-on-merge-queue.test.sh
- name: Load default env
run: |
cp .env.dev.example .env
@@ -413,7 +415,7 @@ jobs:
run: |
pnpm install
- name: run eslint-plugin tests
- run: pnpm --filter @repo/eslint-plugin run test
+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @repo/eslint-plugin run test
tests-in-app-agent-sandbox-runtime:
runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -459,9 +461,9 @@ jobs:
- name: typecheck sandbox runtime
run: pnpm --filter @repo/in-app-agent-sandbox-runtime run typecheck
- name: run sandbox runtime unit tests
- run: pnpm --filter @repo/in-app-agent-sandbox-runtime run test
+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @repo/in-app-agent-sandbox-runtime run test
- name: run sandbox runtime docker e2e tests
- run: pnpm --filter @repo/in-app-agent-sandbox-runtime run test:e2e
+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @repo/in-app-agent-sandbox-runtime run test:e2e
tests-storybook:
runs-on: blacksmith-8vcpu-ubuntu-2404
@@ -575,7 +577,7 @@ jobs:
# The clickhouse binary isn't installed yet, so the SQL-equivalence
# tests skip here (they run for real in the dedicated step below); every
# other shared test hard-fails as usual.
- run: pnpm --filter @langfuse/shared run test
+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @langfuse/shared run test
- name: Install ClickHouse binary for SQL-equivalence tests
# `clickhouse format` (from clickhouse-common-static) normalizes SQL for
# the equivalence snapshots. Its output is version-specific, so pin the
@@ -1384,7 +1386,7 @@ jobs:
cat /tmp/playwright-install.log
exit "$(cat /tmp/playwright-install.exit)"
- name: Run e2e tests
- run: pnpm --filter=web run test:e2e
+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter=web run test:e2e
# Playwright writes a screenshot, an error context and a trace for every
# failure, and the reporter prints the command to open them — but they
# die with the runner, so a flake here is reported with no way to see
@@ -1657,6 +1659,120 @@ jobs:
env:
STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}
+ # A merge-queue run that only went green because a step was retried still hides
+ # a real flake, and the retry means nobody is forced to look at it. Collect the
+ # markers scripts/ci/retry-on-merge-queue.sh leaves in the job logs and report
+ # them, so the flake gets fixed instead of silently re-rolled on every merge.
+ #
+ # Runs after the all-ci-passed gate and is not itself a required check, so it
+ # never delays a merge.
+ notify-flaky-merge-queue:
+ needs: all-ci-passed
+ if: ${{ github.event_name == 'merge_group' && needs.all-ci-passed.outputs.success == 'true' }}
+ runs-on: blacksmith-4vcpu-ubuntu-2404
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ actions: read
+ steps:
+ - name: Collect flaky jobs
+ id: flaky
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPOSITORY: ${{ github.repository }}
+ RUN_ID: ${{ github.run_id }}
+ run: |
+ # grep returns 1 on "no match", which is the normal case here.
+ set +e
+ set +o pipefail
+
+ # Keep in sync with FLAKY_MARKER in scripts/ci/retry-on-merge-queue.sh.
+ # Split so this job's own log — the runner echoes `run:` bodies into
+ # it — can never match the marker and report this job as flaky.
+ MARKER="LANGFUSE_CI_FLAKY""_RETRY_SUCCEEDED"
+ FLAKY=""
+ # A failed lookup must not read as "nothing was retried" — that would
+ # silently drop the very report this job exists to produce. Track it
+ # and fail the job at the end, after reporting whatever was found.
+ INCOMPLETE=0
+
+ JOBS_JSON="$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?per_page=100" --paginate)"
+ if [ "$?" -ne 0 ] || [ -z "$JOBS_JSON" ]; then
+ echo "::error title=Flaky-CI check failed::Could not list this run's jobs; a flaky retry may have gone unreported."
+ echo "flaky=false" >>"$GITHUB_OUTPUT"
+ exit 1
+ fi
+
+ # Only succeeded jobs can have been rescued by a retry; a job that
+ # failed anyway is already reported through the normal failure path.
+ JOBS_TSV="$(jq -r '.jobs[] | select(.conclusion == "success") | "\(.id)\t\(.name)"' <<<"$JOBS_JSON")"
+ if [ "$?" -ne 0 ]; then
+ echo "::error title=Flaky-CI check failed::Could not parse this run's job list; a flaky retry may have gone unreported."
+ echo "flaky=false" >>"$GITHUB_OUTPUT"
+ exit 1
+ fi
+
+ while IFS=$'\t' read -r JOB_ID JOB_NAME; do
+ [ -n "$JOB_ID" ] || continue
+ JOB_LOG="$(gh api "repos/$REPOSITORY/actions/jobs/$JOB_ID/logs" 2>/dev/null)"
+ if [ "$?" -ne 0 ]; then
+ echo "::error title=Flaky-CI check incomplete::Could not download the log for '$JOB_NAME'; if it was retried, that went unreported."
+ INCOMPLETE=1
+ continue
+ fi
+ if grep -qF "$MARKER" <<<"$JOB_LOG"; then
+ if [ -n "$FLAKY" ]; then
+ FLAKY="$(printf '%s\n• %s' "$FLAKY" "$JOB_NAME")"
+ else
+ FLAKY="• $JOB_NAME"
+ fi
+ fi
+ done <<<"$JOBS_TSV"
+
+ if [ -z "$FLAKY" ]; then
+ echo "No retried steps in this run."
+ echo "flaky=false" >>"$GITHUB_OUTPUT"
+ exit "$INCOMPLETE"
+ fi
+
+ echo "Flaky jobs in this run:"
+ printf '%s\n' "$FLAKY"
+ echo "flaky=true" >>"$GITHUB_OUTPUT"
+
+ DELIM="FLAKY_EOF_${RANDOM}${RANDOM}"
+ {
+ echo "jobs<<$DELIM"
+ printf '%s\n' "$FLAKY"
+ echo "$DELIM"
+ } >>"$GITHUB_OUTPUT"
+
+ {
+ echo "### ⚠️ Flaky CI in the merge queue"
+ echo
+ echo "These jobs failed once and passed on retry:"
+ echo
+ printf '%s\n' "$FLAKY"
+ } >>"$GITHUB_STEP_SUMMARY"
+
+ exit "$INCOMPLETE"
+ # always(): an incomplete collection fails the step above on purpose, but
+ # the flakes it did find still need to reach Slack.
+ - name: Notify Slack
+ if: ${{ always() && steps.flaky.outputs.flaky == 'true' }}
+ uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
+ with:
+ webhook: ${{ secrets.SLACK_CI_FAILURE_WORKFLOW_WEBHOOK_URL }}
+ webhook-type: webhook-trigger
+ payload: |
+ title: "⚠️ Flaky CI in merge queue"
+ message: "⚠️ CI passed only after a retry on ${{ github.ref_name }}"
+ ref: "${{ github.ref_name }}"
+ actor: "${{ github.actor }}"
+ event: "${{ github.event_name }}"
+ commit: "${{ github.sha }}"
+ workflow_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ failure_reason: ${{ toJSON(steps.flaky.outputs.jobs) }}
+
build-docker-image-release:
needs: all-ci-passed
# if something inside all-ci-passed was skipped, but everything that ran passed, we still want to deploy
diff --git a/scripts/ci/retry-on-merge-queue.sh b/scripts/ci/retry-on-merge-queue.sh
new file mode 100755
index 0000000000..ce6c97606a
--- /dev/null
+++ b/scripts/ci/retry-on-merge-queue.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+#
+# Run a command and, in the merge queue only, retry it once if it fails.
+#
+# The merge queue watches a single required check (the `all-ci-passed` gate in
+# pipeline.yml). GitHub drops a pull request from the queue the moment that
+# check reports a failure, so a retry is only useful if it happens *before* the
+# job concludes — a `workflow_run` listener that re-runs failed jobs afterwards
+# is too late, the pull request has already left the queue. Hence this wrapper
+# sits inside the job, around the command that actually runs the tests.
+#
+# The retry is deliberately limited to `merge_group`:
+#
+# * On a pull request a failure must stay a failure. Retrying there would
+# hide flakiness from the author, who is the person best placed to fix it.
+# * In the merge queue the code has already been reviewed and the branch has
+# already passed CI once, so a single transient failure costs a full
+# dequeue/requeue cycle for everyone behind it in the queue.
+#
+# A retry that succeeds is not swallowed: the command is announced as flaky on
+# stdout with FLAKY_MARKER, which the `notify-flaky-merge-queue` job greps out
+# of the job logs to report to Slack.
+#
+# Worst case this is no worse than not retrying: if the second attempt fails
+# too, the original exit status is propagated and the job fails exactly as it
+# would have before.
+#
+# The second attempt runs against whatever state the first one left behind —
+# the databases are already seeded and the app is still up. A suite that is not
+# idempotent can therefore fail the retry deterministically, which is why this
+# wraps only suites with no retry of their own rather than everything in sight.
+
+# No `set -e`: the whole point is to observe a non-zero exit and carry on.
+set -uo pipefail
+
+# Keep in sync with the grep in pipeline.yml's `notify-flaky-merge-queue` job.
+readonly FLAKY_MARKER="LANGFUSE_CI_FLAKY_RETRY_SUCCEEDED"
+
+if [[ "$#" -eq 0 ]]; then
+ echo "Usage: retry-on-merge-queue.sh <command> [args...]" >&2
+ exit 2
+fi
+
+"$@"
+status=$?
+
+if [[ "${status}" -eq 0 ]]; then
+ exit 0
+fi
+
+if [[ "${GITHUB_EVENT_NAME:-}" != "merge_group" ]]; then
+ exit "${status}"
+fi
+
+# 128+N means the command was killed by signal N — an OOM kill, or the runner
+# tearing the job down. Re-running the whole suite would hit the same wall and
+# eat the job's remaining time budget, so take the failure as it stands.
+if [[ "${status}" -ge 128 ]]; then
+ echo "::warning title=Not retrying::Command was killed by a signal (exit ${status}); a retry would hit the same limit: $*"
+ exit "${status}"
+fi
+
+echo "::warning title=Retrying a failed merge-queue step::Command failed with exit ${status}; retrying once before failing the merge queue: $*"
+
+"$@"
+retry_status=$?
+
+if [[ "${retry_status}" -ne 0 ]]; then
+ # Report the FIRST attempt's status, so a merge-queue failure exits exactly
+ # as the same failure would on a pull request, where nothing is retried. The
+ # retry's own status is in the message above rather than the exit code.
+ echo "::error title=Merge-queue retry failed::Both attempts failed (exit ${status}, then ${retry_status}): $*"
+ exit "${status}"
+fi
+
+echo "::warning title=Flaky CI::Command failed once and passed on retry: $*"
+echo "${FLAKY_MARKER} job=${GITHUB_JOB:-unknown} command=$*"
+
+if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+ printf '⚠️ Flaky step: `%s` failed with exit %s and passed on retry.\n' "$*" "${status}" >>"${GITHUB_STEP_SUMMARY}"
+fi
+
+# The command passed. Say so explicitly: without this the exit status would be
+# whatever the summary append above returned, so a full disk could fail a step
+# whose tests are green — exactly the dequeue this script exists to prevent.
+exit 0
diff --git a/scripts/ci/retry-on-merge-queue.test.sh b/scripts/ci/retry-on-merge-queue.test.sh
new file mode 100755
index 0000000000..cc5edd505d
--- /dev/null
+++ b/scripts/ci/retry-on-merge-queue.test.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+#
+# Tests for scripts/ci/retry-on-merge-queue.sh.
+#
+# The wrapper decides whether a failing CI step gets a second chance, so the
+# two directions both matter: it must retry inside the merge queue, and it must
+# not retry anywhere else (a flaky pull-request run has to stay visible to its
+# author).
+
+set -uo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+script="$repo_root/scripts/ci/retry-on-merge-queue.sh"
+tmpdir="$(mktemp -d)"
+
+cleanup() {
+ rm -rf "$tmpdir"
+}
+
+trap cleanup EXIT
+
+failures=0
+
+fail() {
+ echo "FAIL: $*" >&2
+ failures=$((failures + 1))
+}
+
+pass() {
+ echo "ok - $*"
+}
+
+# A command that fails for its first `n` invocations and succeeds afterwards,
+# recording every invocation so the call count can be asserted. The retry may
+# be given its own exit code, so a test can tell which attempt's status the
+# wrapper propagated.
+make_command() {
+ local path="$1" fail_times="$2" exit_code="$3" retry_exit_code="${4:-$3}"
+ cat <<EOF >"$path"
+#!/usr/bin/env bash
+count_file="\$(dirname "\$0")/count"
+attempts=\$(( \$(cat "\$count_file" 2>/dev/null || echo 0) + 1 ))
+echo "\$attempts" > "\$count_file"
+if [[ "\$attempts" -eq 1 && "\$attempts" -le $fail_times ]]; then
+ exit $exit_code
+fi
+if [[ "\$attempts" -le $fail_times ]]; then
+ exit $retry_exit_code
+fi
+exit 0
+EOF
+ chmod +x "$path"
+ rm -f "$(dirname "$path")/count"
+}
+
+attempts_for() {
+ cat "$(dirname "$1")/count" 2>/dev/null || echo 0
+}
+
+run_case() {
+ local name="$1" event="$2" fail_times="$3" exit_code="$4"
+ local want_status="$5" want_attempts="$6" want_marker="$7"
+ local retry_exit_code="${8:-$4}"
+
+ local casedir="$tmpdir/$name"
+ mkdir -p "$casedir"
+ local cmd="$casedir/cmd.sh"
+ make_command "$cmd" "$fail_times" "$exit_code" "$retry_exit_code"
+
+ local out status
+ out="$(GITHUB_EVENT_NAME="$event" GITHUB_STEP_SUMMARY="" bash "$script" "$cmd" 2>&1)"
+ status=$?
+
+ local attempts marker="no"
+ attempts="$(attempts_for "$cmd")"
+ if grep -q "LANGFUSE_CI_FLAKY_RETRY_SUCCEEDED" <<<"$out"; then
+ marker="yes"
+ fi
+
+ if [[ "$status" -ne "$want_status" ]]; then
+ fail "$name: exit status was $status, want $want_status"
+ elif [[ "$attempts" -ne "$want_attempts" ]]; then
+ fail "$name: ran the command $attempts time(s), want $want_attempts"
+ elif [[ "$marker" != "$want_marker" ]]; then
+ fail "$name: flaky marker '$marker', want '$want_marker'"
+ else
+ pass "$name"
+ fi
+}
+
+# name event fails exit status attempts marker
+run_case "merge-queue-clean" merge_group 0 1 0 1 no
+run_case "merge-queue-flaky" merge_group 1 1 0 2 yes
+run_case "merge-queue-hard-fail" merge_group 2 3 3 2 no
+
+# When both attempts fail they must exit with the FIRST attempt's status, so a
+# merge-queue failure looks exactly like the same failure on a pull request.
+# The two codes must differ or this cannot tell which one was propagated.
+# name event fails exit status attempts marker retry-exit
+run_case "hard-fail-reports-first-code" merge_group 2 124 124 2 no 1
+run_case "pull-request-no-retry" pull_request 1 1 1 1 no
+run_case "push-no-retry" push 1 1 1 1 no
+run_case "unset-event-no-retry" "" 1 1 1 1 no
+
+# A non-zero exit status must survive the wrapper verbatim, so a job that keys
+# off a specific code still sees it.
+run_case "propagates-exit-status" pull_request 1 42 42 1 no
+
+# A signal death (128+N) is not retried: it is an OOM kill or a job teardown,
+# and a second full run would hit the same limit.
+signal_dir="$tmpdir/signal-case"
+mkdir -p "$signal_dir"
+cat <<'EOF' >"$signal_dir/cmd.sh"
+#!/usr/bin/env bash
+count_file="$(dirname "$0")/count"
+attempts=$(( $(cat "$count_file" 2>/dev/null || echo 0) + 1 ))
+echo "$attempts" > "$count_file"
+if [[ "$attempts" -le 1 ]]; then
+ kill -KILL $$
+fi
+exit 0
+EOF
+chmod +x "$signal_dir/cmd.sh"
+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="" \
+ bash "$script" "$signal_dir/cmd.sh" >/dev/null 2>&1
+signal_status=$?
+signal_attempts="$(cat "$signal_dir/count" 2>/dev/null || echo 0)"
+if [[ "$signal_status" -ne 137 ]]; then
+ fail "signal-death-not-retried: exit status was $signal_status, want 137"
+elif [[ "$signal_attempts" -ne 1 ]]; then
+ fail "signal-death-not-retried: ran $signal_attempts time(s), want 1"
+else
+ pass "signal-death-not-retried"
+fi
+
+# Arguments containing spaces must reach the command intact, not word-split.
+args_dir="$tmpdir/args-case"
+mkdir -p "$args_dir"
+cat <<EOF >"$args_dir/cmd.sh"
+#!/usr/bin/env bash
+printf '%s\n' "\$#" > "$args_dir/argc"
+printf '%s\n' "\$2" > "$args_dir/arg2"
+exit 0
+EOF
+chmod +x "$args_dir/cmd.sh"
+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="" \
+ bash "$script" "$args_dir/cmd.sh" --flag "two words" '*' >/dev/null 2>&1
+if [[ "$(cat "$args_dir/argc")" != "3" ]]; then
+ fail "preserves-argv: command saw $(cat "$args_dir/argc") args, want 3"
+elif [[ "$(cat "$args_dir/arg2")" != "two words" ]]; then
+ fail "preserves-argv: second arg was '$(cat "$args_dir/arg2")', want 'two words'"
+else
+ pass "preserves-argv"
+fi
+
+# A step summary that cannot be written must not turn a passing retry into a
+# failure — the whole point is to keep a green suite in the merge queue.
+unwritable_dir="$tmpdir/unwritable-case"
+mkdir -p "$unwritable_dir"
+make_command "$unwritable_dir/cmd.sh" 1 1
+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="$unwritable_dir/nonexistent-dir/summary.md" \
+ bash "$script" "$unwritable_dir/cmd.sh" >/dev/null 2>&1
+unwritable_status=$?
+if [[ "$unwritable_status" -ne 0 ]]; then
+ fail "unwritable-summary-still-passes: exit status was $unwritable_status, want 0"
+else
+ pass "unwritable-summary-still-passes"
+fi
+
+# No command at all is a usage error, not a silent success.
+if bash "$script" >/dev/null 2>&1; then
+ fail "no-arguments: expected a non-zero exit"
+else
+ status=$?
+ if [[ "$status" -ne 2 ]]; then
+ fail "no-arguments: exit status was $status, want 2"
+ else
+ pass "no-arguments"
+ fi
+fi
+
+# The step summary is appended to only when GitHub provides one.
+summary="$tmpdir/summary.md"
+: >"$summary"
+summary_cmd_dir="$tmpdir/summary-case"
+mkdir -p "$summary_cmd_dir"
+make_command "$summary_cmd_dir/cmd.sh" 1 1
+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="$summary" \
+ bash "$script" "$summary_cmd_dir/cmd.sh" >/dev/null 2>&1
+if grep -q "Flaky step" "$summary"; then
+ pass "writes-step-summary"
+else
+ fail "writes-step-summary: expected a flaky note in \$GITHUB_STEP_SUMMARY"
+fi
+
+if [[ "$failures" -ne 0 ]]; then
+ echo "$failures test(s) failed" >&2
+ exit 1
+fi
+
+echo "All retry-on-merge-queue tests passed."