🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
ci: retry failed test steps once in the merge queue (#18287)
Der Gerätd22b43062641
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.ymlindex 9cd169438b..8733c1e43f 100644--- a/.github/workflows/pipeline.yml+++ b/.github/workflows/pipeline.yml@@ -199,7 +199,7 @@ jobs:- name: Lint Rust targetsrun: cargo clippy --locked --all-targets -- -D warnings- name: Run Rust tests- run: cargo test --locked+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" cargo test --locked- name: Build runtime imagerun: docker build --pull --target runtime -t langfuse-ai-gateway:test .- name: Smoke test runtime image@@ -268,6 +268,8 @@ jobs:run: node --test scripts/agents/sync-agent-shims.test.mjs- name: cursor PR label testsrun: node --test scripts/github/label-cursor-prs.test.mjs+ - name: merge-queue retry helper tests+ run: bash scripts/ci/retry-on-merge-queue.test.sh- name: Load default envrun: |cp .env.dev.example .env@@ -413,7 +415,7 @@ jobs:run: |pnpm install- name: run eslint-plugin tests- run: pnpm --filter @repo/eslint-plugin run test+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @repo/eslint-plugin run testtests-in-app-agent-sandbox-runtime:runs-on: blacksmith-4vcpu-ubuntu-2404@@ -459,9 +461,9 @@ jobs:- name: typecheck sandbox runtimerun: pnpm --filter @repo/in-app-agent-sandbox-runtime run typecheck- name: run sandbox runtime unit tests- run: pnpm --filter @repo/in-app-agent-sandbox-runtime run test+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @repo/in-app-agent-sandbox-runtime run test- name: run sandbox runtime docker e2e tests- run: pnpm --filter @repo/in-app-agent-sandbox-runtime run test:e2e+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @repo/in-app-agent-sandbox-runtime run test:e2etests-storybook:runs-on: blacksmith-8vcpu-ubuntu-2404@@ -575,7 +577,7 @@ jobs:# The clickhouse binary isn't installed yet, so the SQL-equivalence# tests skip here (they run for real in the dedicated step below); every# other shared test hard-fails as usual.- run: pnpm --filter @langfuse/shared run test+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter @langfuse/shared run test- name: Install ClickHouse binary for SQL-equivalence tests# `clickhouse format` (from clickhouse-common-static) normalizes SQL for# the equivalence snapshots. Its output is version-specific, so pin the@@ -1384,7 +1386,7 @@ jobs:cat /tmp/playwright-install.logexit "$(cat /tmp/playwright-install.exit)"- name: Run e2e tests- run: pnpm --filter=web run test:e2e+ run: bash "$GITHUB_WORKSPACE/scripts/ci/retry-on-merge-queue.sh" pnpm --filter=web run test:e2e# Playwright writes a screenshot, an error context and a trace for every# failure, and the reporter prints the command to open them — but they# die with the runner, so a flake here is reported with no way to see@@ -1657,6 +1659,120 @@ jobs:env:STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}+ # A merge-queue run that only went green because a step was retried still hides+ # a real flake, and the retry means nobody is forced to look at it. Collect the+ # markers scripts/ci/retry-on-merge-queue.sh leaves in the job logs and report+ # them, so the flake gets fixed instead of silently re-rolled on every merge.+ #+ # Runs after the all-ci-passed gate and is not itself a required check, so it+ # never delays a merge.+ notify-flaky-merge-queue:+ needs: all-ci-passed+ if: ${{ github.event_name == 'merge_group' && needs.all-ci-passed.outputs.success == 'true' }}+ runs-on: blacksmith-4vcpu-ubuntu-2404+ timeout-minutes: 10+ permissions:+ contents: read+ actions: read+ steps:+ - name: Collect flaky jobs+ id: flaky+ env:+ GH_TOKEN: ${{ github.token }}+ REPOSITORY: ${{ github.repository }}+ RUN_ID: ${{ github.run_id }}+ run: |+ # grep returns 1 on "no match", which is the normal case here.+ set +e+ set +o pipefail++ # Keep in sync with FLAKY_MARKER in scripts/ci/retry-on-merge-queue.sh.+ # Split so this job's own log — the runner echoes `run:` bodies into+ # it — can never match the marker and report this job as flaky.+ MARKER="LANGFUSE_CI_FLAKY""_RETRY_SUCCEEDED"+ FLAKY=""+ # A failed lookup must not read as "nothing was retried" — that would+ # silently drop the very report this job exists to produce. Track it+ # and fail the job at the end, after reporting whatever was found.+ INCOMPLETE=0++ JOBS_JSON="$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?per_page=100" --paginate)"+ if [ "$?" -ne 0 ] || [ -z "$JOBS_JSON" ]; then+ echo "::error title=Flaky-CI check failed::Could not list this run's jobs; a flaky retry may have gone unreported."+ echo "flaky=false" >>"$GITHUB_OUTPUT"+ exit 1+ fi++ # Only succeeded jobs can have been rescued by a retry; a job that+ # failed anyway is already reported through the normal failure path.+ JOBS_TSV="$(jq -r '.jobs[] | select(.conclusion == "success") | "\(.id)\t\(.name)"' <<<"$JOBS_JSON")"+ if [ "$?" -ne 0 ]; then+ echo "::error title=Flaky-CI check failed::Could not parse this run's job list; a flaky retry may have gone unreported."+ echo "flaky=false" >>"$GITHUB_OUTPUT"+ exit 1+ fi++ while IFS=$'\t' read -r JOB_ID JOB_NAME; do+ [ -n "$JOB_ID" ] || continue+ JOB_LOG="$(gh api "repos/$REPOSITORY/actions/jobs/$JOB_ID/logs" 2>/dev/null)"+ if [ "$?" -ne 0 ]; then+ echo "::error title=Flaky-CI check incomplete::Could not download the log for '$JOB_NAME'; if it was retried, that went unreported."+ INCOMPLETE=1+ continue+ fi+ if grep -qF "$MARKER" <<<"$JOB_LOG"; then+ if [ -n "$FLAKY" ]; then+ FLAKY="$(printf '%s\n• %s' "$FLAKY" "$JOB_NAME")"+ else+ FLAKY="• $JOB_NAME"+ fi+ fi+ done <<<"$JOBS_TSV"++ if [ -z "$FLAKY" ]; then+ echo "No retried steps in this run."+ echo "flaky=false" >>"$GITHUB_OUTPUT"+ exit "$INCOMPLETE"+ fi++ echo "Flaky jobs in this run:"+ printf '%s\n' "$FLAKY"+ echo "flaky=true" >>"$GITHUB_OUTPUT"++ DELIM="FLAKY_EOF_${RANDOM}${RANDOM}"+ {+ echo "jobs<<$DELIM"+ printf '%s\n' "$FLAKY"+ echo "$DELIM"+ } >>"$GITHUB_OUTPUT"++ {+ echo "### ⚠️ Flaky CI in the merge queue"+ echo+ echo "These jobs failed once and passed on retry:"+ echo+ printf '%s\n' "$FLAKY"+ } >>"$GITHUB_STEP_SUMMARY"++ exit "$INCOMPLETE"+ # always(): an incomplete collection fails the step above on purpose, but+ # the flakes it did find still need to reach Slack.+ - name: Notify Slack+ if: ${{ always() && steps.flaky.outputs.flaky == 'true' }}+ uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3+ with:+ webhook: ${{ secrets.SLACK_CI_FAILURE_WORKFLOW_WEBHOOK_URL }}+ webhook-type: webhook-trigger+ payload: |+ title: "⚠️ Flaky CI in merge queue"+ message: "⚠️ CI passed only after a retry on ${{ github.ref_name }}"+ ref: "${{ github.ref_name }}"+ actor: "${{ github.actor }}"+ event: "${{ github.event_name }}"+ commit: "${{ github.sha }}"+ workflow_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"+ failure_reason: ${{ toJSON(steps.flaky.outputs.jobs) }}+build-docker-image-release:needs: all-ci-passed# if something inside all-ci-passed was skipped, but everything that ran passed, we still want to deploydiff --git a/scripts/ci/retry-on-merge-queue.sh b/scripts/ci/retry-on-merge-queue.shnew file mode 100755index 0000000000..ce6c97606a--- /dev/null+++ b/scripts/ci/retry-on-merge-queue.sh@@ -0,0 +1,86 @@+#!/usr/bin/env bash+#+# Run a command and, in the merge queue only, retry it once if it fails.+#+# The merge queue watches a single required check (the `all-ci-passed` gate in+# pipeline.yml). GitHub drops a pull request from the queue the moment that+# check reports a failure, so a retry is only useful if it happens *before* the+# job concludes — a `workflow_run` listener that re-runs failed jobs afterwards+# is too late, the pull request has already left the queue. Hence this wrapper+# sits inside the job, around the command that actually runs the tests.+#+# The retry is deliberately limited to `merge_group`:+#+# * On a pull request a failure must stay a failure. Retrying there would+# hide flakiness from the author, who is the person best placed to fix it.+# * In the merge queue the code has already been reviewed and the branch has+# already passed CI once, so a single transient failure costs a full+# dequeue/requeue cycle for everyone behind it in the queue.+#+# A retry that succeeds is not swallowed: the command is announced as flaky on+# stdout with FLAKY_MARKER, which the `notify-flaky-merge-queue` job greps out+# of the job logs to report to Slack.+#+# Worst case this is no worse than not retrying: if the second attempt fails+# too, the original exit status is propagated and the job fails exactly as it+# would have before.+#+# The second attempt runs against whatever state the first one left behind —+# the databases are already seeded and the app is still up. A suite that is not+# idempotent can therefore fail the retry deterministically, which is why this+# wraps only suites with no retry of their own rather than everything in sight.++# No `set -e`: the whole point is to observe a non-zero exit and carry on.+set -uo pipefail++# Keep in sync with the grep in pipeline.yml's `notify-flaky-merge-queue` job.+readonly FLAKY_MARKER="LANGFUSE_CI_FLAKY_RETRY_SUCCEEDED"++if [[ "$#" -eq 0 ]]; then+ echo "Usage: retry-on-merge-queue.sh <command> [args...]" >&2+ exit 2+fi++"$@"+status=$?++if [[ "${status}" -eq 0 ]]; then+ exit 0+fi++if [[ "${GITHUB_EVENT_NAME:-}" != "merge_group" ]]; then+ exit "${status}"+fi++# 128+N means the command was killed by signal N — an OOM kill, or the runner+# tearing the job down. Re-running the whole suite would hit the same wall and+# eat the job's remaining time budget, so take the failure as it stands.+if [[ "${status}" -ge 128 ]]; then+ echo "::warning title=Not retrying::Command was killed by a signal (exit ${status}); a retry would hit the same limit: $*"+ exit "${status}"+fi++echo "::warning title=Retrying a failed merge-queue step::Command failed with exit ${status}; retrying once before failing the merge queue: $*"++"$@"+retry_status=$?++if [[ "${retry_status}" -ne 0 ]]; then+ # Report the FIRST attempt's status, so a merge-queue failure exits exactly+ # as the same failure would on a pull request, where nothing is retried. The+ # retry's own status is in the message above rather than the exit code.+ echo "::error title=Merge-queue retry failed::Both attempts failed (exit ${status}, then ${retry_status}): $*"+ exit "${status}"+fi++echo "::warning title=Flaky CI::Command failed once and passed on retry: $*"+echo "${FLAKY_MARKER} job=${GITHUB_JOB:-unknown} command=$*"++if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then+ printf '⚠️ Flaky step: `%s` failed with exit %s and passed on retry.\n' "$*" "${status}" >>"${GITHUB_STEP_SUMMARY}"+fi++# The command passed. Say so explicitly: without this the exit status would be+# whatever the summary append above returned, so a full disk could fail a step+# whose tests are green — exactly the dequeue this script exists to prevent.+exit 0diff --git a/scripts/ci/retry-on-merge-queue.test.sh b/scripts/ci/retry-on-merge-queue.test.shnew file mode 100755index 0000000000..cc5edd505d--- /dev/null+++ b/scripts/ci/retry-on-merge-queue.test.sh@@ -0,0 +1,201 @@+#!/usr/bin/env bash+#+# Tests for scripts/ci/retry-on-merge-queue.sh.+#+# The wrapper decides whether a failing CI step gets a second chance, so the+# two directions both matter: it must retry inside the merge queue, and it must+# not retry anywhere else (a flaky pull-request run has to stay visible to its+# author).++set -uo pipefail++repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+script="$repo_root/scripts/ci/retry-on-merge-queue.sh"+tmpdir="$(mktemp -d)"++cleanup() {+ rm -rf "$tmpdir"+}++trap cleanup EXIT++failures=0++fail() {+ echo "FAIL: $*" >&2+ failures=$((failures + 1))+}++pass() {+ echo "ok - $*"+}++# A command that fails for its first `n` invocations and succeeds afterwards,+# recording every invocation so the call count can be asserted. The retry may+# be given its own exit code, so a test can tell which attempt's status the+# wrapper propagated.+make_command() {+ local path="$1" fail_times="$2" exit_code="$3" retry_exit_code="${4:-$3}"+ cat <<EOF >"$path"+#!/usr/bin/env bash+count_file="\$(dirname "\$0")/count"+attempts=\$(( \$(cat "\$count_file" 2>/dev/null || echo 0) + 1 ))+echo "\$attempts" > "\$count_file"+if [[ "\$attempts" -eq 1 && "\$attempts" -le $fail_times ]]; then+ exit $exit_code+fi+if [[ "\$attempts" -le $fail_times ]]; then+ exit $retry_exit_code+fi+exit 0+EOF+ chmod +x "$path"+ rm -f "$(dirname "$path")/count"+}++attempts_for() {+ cat "$(dirname "$1")/count" 2>/dev/null || echo 0+}++run_case() {+ local name="$1" event="$2" fail_times="$3" exit_code="$4"+ local want_status="$5" want_attempts="$6" want_marker="$7"+ local retry_exit_code="${8:-$4}"++ local casedir="$tmpdir/$name"+ mkdir -p "$casedir"+ local cmd="$casedir/cmd.sh"+ make_command "$cmd" "$fail_times" "$exit_code" "$retry_exit_code"++ local out status+ out="$(GITHUB_EVENT_NAME="$event" GITHUB_STEP_SUMMARY="" bash "$script" "$cmd" 2>&1)"+ status=$?++ local attempts marker="no"+ attempts="$(attempts_for "$cmd")"+ if grep -q "LANGFUSE_CI_FLAKY_RETRY_SUCCEEDED" <<<"$out"; then+ marker="yes"+ fi++ if [[ "$status" -ne "$want_status" ]]; then+ fail "$name: exit status was $status, want $want_status"+ elif [[ "$attempts" -ne "$want_attempts" ]]; then+ fail "$name: ran the command $attempts time(s), want $want_attempts"+ elif [[ "$marker" != "$want_marker" ]]; then+ fail "$name: flaky marker '$marker', want '$want_marker'"+ else+ pass "$name"+ fi+}++# name event fails exit status attempts marker+run_case "merge-queue-clean" merge_group 0 1 0 1 no+run_case "merge-queue-flaky" merge_group 1 1 0 2 yes+run_case "merge-queue-hard-fail" merge_group 2 3 3 2 no++# When both attempts fail they must exit with the FIRST attempt's status, so a+# merge-queue failure looks exactly like the same failure on a pull request.+# The two codes must differ or this cannot tell which one was propagated.+# name event fails exit status attempts marker retry-exit+run_case "hard-fail-reports-first-code" merge_group 2 124 124 2 no 1+run_case "pull-request-no-retry" pull_request 1 1 1 1 no+run_case "push-no-retry" push 1 1 1 1 no+run_case "unset-event-no-retry" "" 1 1 1 1 no++# A non-zero exit status must survive the wrapper verbatim, so a job that keys+# off a specific code still sees it.+run_case "propagates-exit-status" pull_request 1 42 42 1 no++# A signal death (128+N) is not retried: it is an OOM kill or a job teardown,+# and a second full run would hit the same limit.+signal_dir="$tmpdir/signal-case"+mkdir -p "$signal_dir"+cat <<'EOF' >"$signal_dir/cmd.sh"+#!/usr/bin/env bash+count_file="$(dirname "$0")/count"+attempts=$(( $(cat "$count_file" 2>/dev/null || echo 0) + 1 ))+echo "$attempts" > "$count_file"+if [[ "$attempts" -le 1 ]]; then+ kill -KILL $$+fi+exit 0+EOF+chmod +x "$signal_dir/cmd.sh"+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="" \+ bash "$script" "$signal_dir/cmd.sh" >/dev/null 2>&1+signal_status=$?+signal_attempts="$(cat "$signal_dir/count" 2>/dev/null || echo 0)"+if [[ "$signal_status" -ne 137 ]]; then+ fail "signal-death-not-retried: exit status was $signal_status, want 137"+elif [[ "$signal_attempts" -ne 1 ]]; then+ fail "signal-death-not-retried: ran $signal_attempts time(s), want 1"+else+ pass "signal-death-not-retried"+fi++# Arguments containing spaces must reach the command intact, not word-split.+args_dir="$tmpdir/args-case"+mkdir -p "$args_dir"+cat <<EOF >"$args_dir/cmd.sh"+#!/usr/bin/env bash+printf '%s\n' "\$#" > "$args_dir/argc"+printf '%s\n' "\$2" > "$args_dir/arg2"+exit 0+EOF+chmod +x "$args_dir/cmd.sh"+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="" \+ bash "$script" "$args_dir/cmd.sh" --flag "two words" '*' >/dev/null 2>&1+if [[ "$(cat "$args_dir/argc")" != "3" ]]; then+ fail "preserves-argv: command saw $(cat "$args_dir/argc") args, want 3"+elif [[ "$(cat "$args_dir/arg2")" != "two words" ]]; then+ fail "preserves-argv: second arg was '$(cat "$args_dir/arg2")', want 'two words'"+else+ pass "preserves-argv"+fi++# A step summary that cannot be written must not turn a passing retry into a+# failure — the whole point is to keep a green suite in the merge queue.+unwritable_dir="$tmpdir/unwritable-case"+mkdir -p "$unwritable_dir"+make_command "$unwritable_dir/cmd.sh" 1 1+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="$unwritable_dir/nonexistent-dir/summary.md" \+ bash "$script" "$unwritable_dir/cmd.sh" >/dev/null 2>&1+unwritable_status=$?+if [[ "$unwritable_status" -ne 0 ]]; then+ fail "unwritable-summary-still-passes: exit status was $unwritable_status, want 0"+else+ pass "unwritable-summary-still-passes"+fi++# No command at all is a usage error, not a silent success.+if bash "$script" >/dev/null 2>&1; then+ fail "no-arguments: expected a non-zero exit"+else+ status=$?+ if [[ "$status" -ne 2 ]]; then+ fail "no-arguments: exit status was $status, want 2"+ else+ pass "no-arguments"+ fi+fi++# The step summary is appended to only when GitHub provides one.+summary="$tmpdir/summary.md"+: >"$summary"+summary_cmd_dir="$tmpdir/summary-case"+mkdir -p "$summary_cmd_dir"+make_command "$summary_cmd_dir/cmd.sh" 1 1+GITHUB_EVENT_NAME=merge_group GITHUB_STEP_SUMMARY="$summary" \+ bash "$script" "$summary_cmd_dir/cmd.sh" >/dev/null 2>&1+if grep -q "Flaky step" "$summary"; then+ pass "writes-step-summary"+else+ fail "writes-step-summary: expected a flaky note in \$GITHUB_STEP_SUMMARY"+fi++if [[ "$failures" -ne 0 ]]; then+ echo "$failures test(s) failed" >&2+ exit 1+fi++echo "All retry-on-merge-queue tests passed."