Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ci): skip Slack failure alert for cancelled runs and name cancelled jobs (#19605)

Tobias Wochinger6e0def067d0f
diff --git a/.github/actions/notify-slack-failure/action.yml b/.github/actions/notify-slack-failure/action.yml
index 4eb0d9b0b6..41168efe0b 100644
--- a/.github/actions/notify-slack-failure/action.yml
+++ b/.github/actions/notify-slack-failure/action.yml
@@ -8,6 +8,10 @@ inputs:
webhook-url:
description: Slack Workflow webhook URL.
required: true
+ ignore-job:
+ description: Name of a job to leave out of the failure reason, e.g. a gate job that fails itself whenever one of its dependencies fails.
+ required: false
+ default: ""
outputs:
reason:
@@ -24,6 +28,7 @@ runs:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
+ IGNORE_JOB: ${{ inputs.ignore-job }}
run: |
# GitHub Actions runs composite `bash` steps with `-e -o pipefail`.
# Several greps below intentionally return 1 on "no match" (e.g. no
@@ -34,9 +39,13 @@ runs:
set +e
set +o pipefail
+ # Cancelled jobs are included because a job that exceeds its
+ # timeout-minutes concludes as `cancelled`, not `failure`. Failed jobs
+ # sort first so the MAX_JOBS cap below keeps them.
FAILED_JOBS_JSON="$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?per_page=100" --paginate \
- | jq -sc --arg self_job "$GITHUB_JOB" \
- '[.[].jobs[] | select(.conclusion == "failure" and .name != $self_job)]')"
+ | jq -sc --arg self_job "$GITHUB_JOB" --arg ignore_job "$IGNORE_JOB" \
+ '[.[].jobs[] | select((.conclusion == "failure" or .conclusion == "cancelled") and .name != $self_job and .name != $ignore_job)]
+ | sort_by(.conclusion != "failure")')"
FAILED_COUNT=$(jq 'length' <<<"$FAILED_JOBS_JSON")
MAX_JOBS=3
MAX_EXCERPT_LEN=250
@@ -46,14 +55,14 @@ runs:
# Cap how many failed jobs we pull logs for — a GitHub-wide outage can
# fail a dozen unrelated jobs at once, and we don't want to balloon the
# Slack message or the API call count for that case. Excludes the
- # current job itself: the gate jobs that call this action
- # (all-ci-passed, notify-docker-image-release) deliberately fail
- # themselves to propagate branch-protection status, which would
- # otherwise always show up as meaningless noise.
+ # current job and `ignore-job`: a gate job such as all-ci-passed
+ # deliberately fails itself to propagate branch-protection status,
+ # which would otherwise always show up as meaningless noise.
while IFS= read -r job; do
JOB_ID=$(jq -r '.id' <<<"$job")
JOB_NAME=$(jq -r '.name' <<<"$job")
- FAILED_STEPS=$(jq -r '[.steps[]? | select(.conclusion == "failure") | .name] | join(", ")' <<<"$job")
+ [ "$(jq -r '.conclusion' <<<"$job")" = "cancelled" ] && JOB_NAME="$JOB_NAME (cancelled, e.g. timed out)"
+ FAILED_STEPS=$(jq -r '[.steps[]? | select(.conclusion == "failure" or .conclusion == "cancelled") | .name] | join(", ")' <<<"$job")
JOB_LOG="$(gh api "repos/$REPOSITORY/actions/jobs/$JOB_ID/logs" 2>/dev/null || true)"
@@ -110,11 +119,11 @@ runs:
done < <(jq -c ".[0:$MAX_JOBS][]" <<<"$FAILED_JOBS_JSON")
if [ "$FAILED_COUNT" -gt "$MAX_JOBS" ]; then
- REASON="$(printf '%s\n\n…and %s more failed job(s)' "$REASON" "$((FAILED_COUNT - MAX_JOBS))")"
+ REASON="$(printf '%s\n\n…and %s more failed or cancelled job(s)' "$REASON" "$((FAILED_COUNT - MAX_JOBS))")"
fi
if [ -z "$REASON" ]; then
- REASON="Unknown (no failed job found via the Actions API — check the workflow run)"
+ REASON="Unknown (no failed or cancelled job found via the Actions API — check the workflow run)"
fi
max_len=1200
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 1c3a58a827..9df55fd85e 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -1741,7 +1741,6 @@ jobs:
if: always()
permissions:
contents: read
- actions: read
outputs:
success: ${{ steps.set-success-output.outputs.success }}
steps:
@@ -1761,21 +1760,33 @@ jobs:
if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
run: exit 1
working-directory: .
+ - name: Output job results
+ run: |
+ echo "Job results: ${STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS}"
+ env:
+ STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}
+
+ # The alert lives in its own job because only a job-level `cancelled()` tells
+ # whether the run was cancelled. Inside a step of the always() gate it reads
+ # the gate's own status, which never becomes cancelled. A job that exceeds
+ # timeout-minutes is cancelled without cancelling the run, so it still alerts.
+ notify-ci-failure:
+ needs: all-ci-passed
+ if: ${{ !cancelled() && needs.all-ci-passed.result == 'failure' && github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v3' || startsWith(github.ref, 'refs/tags/')) }}
+ runs-on: blacksmith-4vcpu-ubuntu-2404
+ permissions:
+ contents: read
+ actions: read
+ steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- if: failure() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v3' || startsWith(github.ref, 'refs/tags/'))
with:
persist-credentials: false
- name: Notify Slack
- if: failure() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/v3' || startsWith(github.ref, 'refs/tags/'))
uses: ./.github/actions/notify-slack-failure
with:
message: "❌ CI failed on ${{ github.ref_name }}"
webhook-url: ${{ secrets.SLACK_CI_FAILURE_WORKFLOW_WEBHOOK_URL }}
- - name: Output job results
- run: |
- echo "Job results: ${STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS}"
- env:
- STEPS_SET_SUCCESS_OUTPUT_OUTPUTS_SUCCESS: ${{ steps.set-success-output.outputs.success }}
+ ignore-job: all-ci-passed
# A merge-queue run that only went green because a step was retried still hides
# a real flake, and the retry means nobody is forced to look at it. Collect the
@@ -2256,21 +2267,17 @@ jobs:
needs:
- build-docker-image-release
- publish-docker-image-release
- if: always() && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+ # Job-level `!cancelled()` skips whole-run cancels; see notify-ci-failure.
+ if: ${{ !cancelled() && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
runs-on: blacksmith-4vcpu-ubuntu-2404
permissions:
contents: read
actions: read
steps:
- - name: Fail when a release image job failed
- if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
- run: exit 1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- if: failure()
with:
persist-credentials: false
- name: Notify Slack
- if: failure()
uses: ./.github/actions/notify-slack-failure
with:
message: "❌ Docker release failed on ${{ github.ref_name }}"