Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Re-run a blocked stable desktop release once before paging (#5824)

Andrew Mikofalvycf9b84cd0dbc
diff --git a/.github/scripts/cache-mode-shape.test.mjs b/.github/scripts/cache-mode-shape.test.mjs
index 29df7ab56..2c42dc6d1 100644
--- a/.github/scripts/cache-mode-shape.test.mjs
+++ b/.github/scripts/cache-mode-shape.test.mjs
@@ -205,6 +205,7 @@ describe('credentialed jobs in public/open-knowledge/.github/workflows declare t
'bug-lane.yml#bug-lane': 'none',
'desktop-build-win-linux.yml#build-windows': 'none',
'desktop-build.yml#build-macos-dmg': 'none',
+ 'desktop-release-auto-retry.yml#rerun': 'none',
'desktop-release-draft-janitor.yml#sweep': 'none',
'desktop-release.yml#prepare': 'none',
'desktop-release.yml#build-macos': 'none',
diff --git a/.github/scripts/decide-release-retry.mjs b/.github/scripts/decide-release-retry.mjs
new file mode 100644
index 000000000..e673fe6e7
--- /dev/null
+++ b/.github/scripts/decide-release-retry.mjs
@@ -0,0 +1,190 @@
+/* biome-ignore-all lint/suspicious/noUndeclaredEnvVars: GitHub Actions invokes this entrypoint directly, outside Turbo. */
+import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { compareVersions } from './write-back-gate.mjs';
+
+export const RETRYABLE_OUTCOMES = Object.freeze([
+ 'transient-exhausted',
+ 'deadline',
+ 'signal',
+ 'attempt-timeout',
+]);
+
+const JOB_TIMEOUT = /^The job has exceeded the maximum execution time of /;
+
+const RETRYABLE_INFRASTRUCTURE = Object.freeze([
+ /^The hosted runner lost communication with the server\./,
+ /^Failed to FinalizeArtifact\b/,
+ /gh release upload failed after \d+ attempts/,
+ JOB_TIMEOUT,
+]);
+
+const PLATFORM_JOBS = Object.freeze([
+ ['mac', /^build-macos\b/],
+ ['windows', /^build-windows\b/],
+ ['linux', /^build-linux\b/],
+]);
+const FAILED_CONCLUSIONS = new Set(['failure', 'cancelled', 'timed_out']);
+const TIMEOUT_CANCELLATION = 'The operation was canceled.';
+const GENERIC_EXIT = /^Process completed with exit code \d+\.$/;
+const STOP_LINE = /\bdecision=stop reason=(\S+) outcome=(\S+)/;
+
+function firstLine(message) {
+ return String(message).split('\n')[0].slice(0, 160);
+}
+
+function judgeFailure(message) {
+ if (GENERIC_EXIT.test(message)) return { kind: 'generic' };
+ const stop = STOP_LINE.exec(message);
+ if (stop) {
+ const [, reason, outcome] = stop;
+ const cause = `outcome=${outcome} reason=${reason}`;
+ const retryable =
+ RETRYABLE_OUTCOMES.includes(outcome) ||
+ (outcome === 'terminal' && reason === 'rule:download-integrity');
+ return { kind: retryable ? 'retryable' : 'blocking', cause };
+ }
+ if (RETRYABLE_INFRASTRUCTURE.some((pattern) => pattern.test(message))) {
+ return { kind: 'retryable', cause: firstLine(message) };
+ }
+ return { kind: 'blocking', cause: firstLine(message) };
+}
+
+export function classifyFailedJob(annotations) {
+ const messages = (annotations ?? [])
+ .filter((annotation) => annotation.level === 'failure')
+ .map((annotation) => annotation.message);
+ const timedOut = messages.some((message) => JOB_TIMEOUT.test(message));
+ const judged = messages.map((message) => {
+ if (timedOut && message === TIMEOUT_CANCELLATION) return { kind: 'generic' };
+ return judgeFailure(message);
+ });
+ const blocking = judged.find((judgement) => judgement.kind === 'blocking');
+ if (blocking) return { retryable: false, cause: blocking.cause };
+ const retryable = judged.find((judgement) => judgement.kind === 'retryable');
+ if (retryable) return { retryable: true, cause: retryable.cause };
+ return { retryable: false, cause: 'no recognized cause in its annotations' };
+}
+
+function isRequired(jobName, requiredPlatforms) {
+ const platform = PLATFORM_JOBS.find(([, pattern]) => pattern.test(jobName))?.[0];
+ return platform === undefined || requiredPlatforms.includes(platform);
+}
+
+export function decideRetry({
+ tag,
+ channel,
+ runAttempt,
+ smokeVerdict,
+ latestStable,
+ jobs,
+ requiredPlatforms = ['mac', 'windows', 'linux'],
+}) {
+ const page = (reason) => ({ action: 'page', reason });
+ if (channel !== 'latest') return page(`channel ${channel || 'unknown'} is not a stable cut`);
+ if (smokeVerdict === 'fail') return page('the packaged app failed the DMG smoke');
+ if (latestStable && compareVersions(latestStable, tag) > 0) {
+ return page(`a newer stable ${latestStable} has already shipped`);
+ }
+ if (runAttempt > 1) return page(`attempt ${runAttempt} is already a re-run`);
+ const failed = (jobs ?? []).filter(
+ (job) => FAILED_CONCLUSIONS.has(job.conclusion) && isRequired(job.name, requiredPlatforms),
+ );
+ if (failed.length === 0) return page('no failed job to re-run');
+ const judged = failed.map((job) => ({ name: job.name, ...classifyFailedJob(job.annotations) }));
+ const blocking = judged.filter((job) => !job.retryable);
+ if (blocking.length > 0) {
+ return page(blocking.map((job) => `${job.name}: ${job.cause}`).join('; '));
+ }
+ return {
+ action: 'retry',
+ reason: judged.map((job) => `${job.name}: ${job.cause}`).join('; '),
+ };
+}
+
+async function github(path, token, { allowNotFound = false } = {}) {
+ const api = process.env.GITHUB_API_URL || 'https://api.github.com';
+ const response = await fetch(`${api}${path}`, {
+ headers: {
+ accept: 'application/vnd.github+json',
+ authorization: `Bearer ${token}`,
+ 'x-github-api-version': '2022-11-28',
+ },
+ signal: AbortSignal.timeout(30_000),
+ });
+ if (allowNotFound && response.status === 404) return null;
+ if (!response.ok) throw new Error(`GET ${path} failed (${response.status})`);
+ return response.json();
+}
+
+async function readJobs({ repo, runId, runAttempt, token }) {
+ const jobs = [];
+ for (let page = 1; ; page += 1) {
+ const body = await github(
+ `/repos/${repo}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100&page=${page}`,
+ token,
+ );
+ jobs.push(...body.jobs);
+ if (jobs.length >= body.total_count || body.jobs.length === 0) break;
+ }
+ return Promise.all(
+ jobs.map(async (job) => ({
+ name: job.name,
+ conclusion: job.conclusion,
+ annotations: FAILED_CONCLUSIONS.has(job.conclusion)
+ ? (await github(`/repos/${repo}/check-runs/${job.id}/annotations?per_page=100`, token)).map(
+ (annotation) => ({ level: annotation.annotation_level, message: annotation.message }),
+ )
+ : [],
+ })),
+ );
+}
+
+async function main() {
+ const repo = process.env.GITHUB_REPOSITORY;
+ const runId = process.env.GITHUB_RUN_ID;
+ const runAttempt = Number(process.env.GITHUB_RUN_ATTEMPT || '1');
+ const token = process.env.GH_TOKEN;
+ const tag = process.env.RELEASE_TAG;
+ if (!repo || !runId || !token || !tag) {
+ throw new Error('GITHUB_REPOSITORY, GITHUB_RUN_ID, GH_TOKEN and RELEASE_TAG are required');
+ }
+ const latest = await github(`/repos/${repo}/releases/latest`, token, { allowNotFound: true });
+ const decision = decideRetry({
+ tag,
+ channel: process.env.CHANNEL,
+ runAttempt,
+ smokeVerdict: process.env.SMOKE_VERDICT,
+ latestStable: latest?.tag_name ?? null,
+ jobs: await readJobs({ repo, runId, runAttempt, token }),
+ requiredPlatforms: process.env.REQUIRED_PLATFORMS
+ ? process.env.REQUIRED_PLATFORMS.split(',').map((platform) => platform.trim())
+ : undefined,
+ });
+ const reason = decision.reason.replace(/\s+/g, ' ');
+ if (decision.action === 'retry') {
+ console.log(
+ `::warning::RELEASE RETRYING: ${tag} — the failed jobs re-run once automatically; a second failure pages. ${reason}`,
+ );
+ } else {
+ console.log(`::notice::No automatic re-run for ${tag}: ${reason}`);
+ }
+ if (process.env.DECISION_PATH) {
+ mkdirSync(dirname(process.env.DECISION_PATH), { recursive: true });
+ writeFileSync(
+ process.env.DECISION_PATH,
+ JSON.stringify({ tag, runId, runAttempt, ...decision, reason }),
+ );
+ }
+ if (process.env.GITHUB_OUTPUT) {
+ appendFileSync(process.env.GITHUB_OUTPUT, `action=${decision.action}\nreason=${reason}\n`);
+ }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ main().catch((error) => {
+ console.log(`::warning::Automatic re-run decision failed; paging instead. ${error.message}`);
+ process.exitCode = 1;
+ });
+}
diff --git a/.github/scripts/decide-release-retry.test.mjs b/.github/scripts/decide-release-retry.test.mjs
new file mode 100644
index 000000000..bb81b6e0c
--- /dev/null
+++ b/.github/scripts/decide-release-retry.test.mjs
@@ -0,0 +1,273 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterAll, describe, expect, test } from 'vitest';
+import { parse } from 'yaml';
+import { classifyFailedJob, decideRetry, RETRYABLE_OUTCOMES } from './decide-release-retry.mjs';
+import { STOP_OUTCOMES } from './retry-transient.mjs';
+
+const failure = (message) => ({ level: 'failure', message });
+const notice = (message) => ({ level: 'notice', message });
+const EXIT_1 = failure('Process completed with exit code 1.');
+const stop = (outcome, reason = 'rule:timestamp-missing') =>
+ failure(
+ `electron-builder (macOS) decision=stop reason=${reason} outcome=${outcome} attempt=3/3 code=1 signal=none`,
+ );
+
+const stable = {
+ tag: 'v0.83.0',
+ channel: 'latest',
+ runAttempt: 1,
+ smokeVerdict: '',
+ latestStable: 'v0.82.3',
+};
+const macFailed = (annotations) => [
+ { name: 'prepare', conclusion: 'success', annotations: [] },
+ { name: 'build-macos (stable)', conclusion: 'failure', annotations },
+ { name: 'build-windows (stable)', conclusion: 'success', annotations: [] },
+ { name: 'publish-assets', conclusion: 'skipped', annotations: [] },
+];
+
+describe('classifyFailedJob', () => {
+ test.each([
+ [
+ 'an Apple notarization attempt timeout',
+ [EXIT_1, stop('attempt-timeout', 'control:attempt-timeout')],
+ ],
+ ['an exhausted transient Apple error', [EXIT_1, stop('transient-exhausted')]],
+ ['a wrapper deadline', [EXIT_1, stop('deadline', 'control:deadline')]],
+ ['a corrupted download', [EXIT_1, stop('terminal', 'rule:download-integrity')]],
+ [
+ 'a lost hosted runner',
+ [
+ failure(
+ 'The hosted runner lost communication with the server. Anything in your workflow that terminates the runner process can cause this error.',
+ ),
+ ],
+ ],
+ [
+ 'an artifact upload timeout',
+ [failure('Failed to FinalizeArtifact: Unable to make request: ETIMEDOUT')],
+ ],
+ ['a release upload outage', [failure('gh release upload failed after 3 attempts.')]],
+ [
+ 'a job timeout',
+ [
+ failure('The job has exceeded the maximum execution time of 45m0s'),
+ failure('The operation was canceled.'),
+ ],
+ ],
+ ])('re-runs %s', (_label, annotations) => {
+ expect(classifyFailedJob(annotations).retryable).toBe(true);
+ });
+
+ test.each([
+ ['a terminal packaging error', [EXIT_1, stop('terminal', 'rule:missing-executable')]],
+ [
+ 'an unknown error that already used its retry',
+ [EXIT_1, stop('unknown-exhausted', 'diagnostic:unknown')],
+ ],
+ [
+ 'a failure with no recognized cause',
+ [EXIT_1, notice('Signing creds present — running signed build.')],
+ ],
+ [
+ 'a broken toolchain step',
+ [
+ failure(
+ 'Error: The requested pnpm version "10.33.0" resolved to 10.33.0, but this action only installs pnpm v11 or newer.',
+ ),
+ ],
+ ],
+ [
+ 'a retryable cause beside an unrecognized error',
+ [
+ stop('transient-exhausted'),
+ failure('No .dmg found under packages/desktop/dist-desktop for v0.83.0'),
+ ],
+ ],
+ [
+ 'a human cancellation',
+ [failure('The run was canceled by @someone.'), failure('The operation was canceled.')],
+ ],
+ ])('pages on %s', (_label, annotations) => {
+ expect(classifyFailedJob(annotations).retryable).toBe(false);
+ });
+
+ test('every wrapper stop outcome is either re-run or deliberately paged', () => {
+ const paged = STOP_OUTCOMES.filter((outcome) => !RETRYABLE_OUTCOMES.includes(outcome));
+ expect(paged).toEqual([
+ 'terminal',
+ 'unknown-exhausted',
+ 'child-signal',
+ 'spawn-failure',
+ 'cleanup-failure',
+ ]);
+ expect(RETRYABLE_OUTCOMES.every((outcome) => STOP_OUTCOMES.includes(outcome))).toBe(true);
+ });
+});
+
+describe('decideRetry', () => {
+ const timedOut = macFailed([EXIT_1, stop('attempt-timeout', 'control:attempt-timeout')]);
+
+ test('re-runs a first stable attempt whose every failure is retryable', () => {
+ expect(decideRetry({ ...stable, jobs: timedOut })).toEqual({
+ action: 'retry',
+ reason: 'build-macos (stable): outcome=attempt-timeout reason=control:attempt-timeout',
+ });
+ });
+
+ test('re-runs when several platforms each failed retryably', () => {
+ const jobs = [
+ ...timedOut,
+ {
+ name: 'build-linux (stable, ubuntu-latest)',
+ conclusion: 'failure',
+ annotations: [failure('Failed to FinalizeArtifact: Unable to make request: ETIMEDOUT')],
+ },
+ ];
+ expect(decideRetry({ ...stable, jobs }).action).toBe('retry');
+ });
+
+ test('pages when any one failed job is not retryable, and names it', () => {
+ const jobs = [
+ ...timedOut,
+ {
+ name: 'build-windows (stable)',
+ conclusion: 'failure',
+ annotations: [EXIT_1, stop('terminal', 'rule:missing-executable')],
+ },
+ ];
+ expect(decideRetry({ ...stable, jobs })).toEqual({
+ action: 'page',
+ reason: 'build-windows (stable): outcome=terminal reason=rule:missing-executable',
+ });
+ });
+
+ test('pages on the second attempt, so the automatic re-run happens once', () => {
+ expect(decideRetry({ ...stable, runAttempt: 2, jobs: timedOut })).toEqual({
+ action: 'page',
+ reason: 'attempt 2 is already a re-run',
+ });
+ });
+
+ test('names a newer stable or a failed smoke on attempt 2 instead of the attempt count', () => {
+ expect(
+ decideRetry({ ...stable, runAttempt: 2, latestStable: 'v0.84.0', jobs: timedOut }).reason,
+ ).toBe('a newer stable v0.84.0 has already shipped');
+ expect(
+ decideRetry({ ...stable, runAttempt: 2, smokeVerdict: 'fail', jobs: timedOut }).reason,
+ ).toBe('the packaged app failed the DMG smoke');
+ });
+
+ test('pages on a real smoke failure even beside a retryable cause', () => {
+ expect(decideRetry({ ...stable, smokeVerdict: 'fail', jobs: timedOut }).action).toBe('page');
+ });
+
+ test('never re-runs a tag a newer stable has superseded', () => {
+ expect(decideRetry({ ...stable, latestStable: 'v0.84.0', jobs: timedOut })).toEqual({
+ action: 'page',
+ reason: 'a newer stable v0.84.0 has already shipped',
+ });
+ });
+
+ test('re-runs when no stable has shipped yet', () => {
+ expect(decideRetry({ ...stable, latestStable: null, jobs: timedOut }).action).toBe('retry');
+ });
+
+ test('leaves beta cuts alone', () => {
+ expect(decideRetry({ ...stable, channel: 'beta', jobs: timedOut }).action).toBe('page');
+ });
+
+ test('pages when finalize did not succeed but no job failed', () => {
+ expect(decideRetry({ ...stable, jobs: macFailed([]).slice(0, 1) })).toEqual({
+ action: 'page',
+ reason: 'no failed job to re-run',
+ });
+ });
+
+ test('ignores a failed platform the cut does not require', () => {
+ const jobs = [
+ ...timedOut,
+ {
+ name: 'build-windows (stable)',
+ conclusion: 'failure',
+ annotations: [EXIT_1, stop('terminal', 'rule:missing-executable')],
+ },
+ ];
+ expect(decideRetry({ ...stable, jobs, requiredPlatforms: ['mac', 'linux'] }).action).toBe(
+ 'retry',
+ );
+ expect(decideRetry({ ...stable, jobs }).action).toBe('page');
+ });
+
+ test('re-runs a timed-out job, which GitHub reports as cancelled', () => {
+ const jobs = [
+ {
+ name: 'build-linux (stable, ubuntu-latest)',
+ conclusion: 'cancelled',
+ annotations: [
+ failure('The job has exceeded the maximum execution time of 45m0s'),
+ failure('The operation was canceled.'),
+ ],
+ },
+ ];
+ expect(decideRetry({ ...stable, jobs }).action).toBe('retry');
+ });
+
+ test('counts a cancelled job as failed', () => {
+ const jobs = [{ name: 'build-macos (stable)', conclusion: 'cancelled', annotations: [EXIT_1] }];
+ expect(decideRetry({ ...stable, jobs }).action).toBe('page');
+ });
+});
+
+describe('finalize refuses to promote a superseded stable on a re-run', () => {
+ const finalize = parse(
+ readFileSync(
+ join(dirname(fileURLToPath(import.meta.url)), '..', 'workflows', 'desktop-release.yml'),
+ 'utf8',
+ ),
+ ).jobs.finalize;
+ const step = finalize.steps.find(
+ (candidate) => candidate.name === 'Refuse to promote a superseded stable on a re-run',
+ );
+ const dir = mkdtempSync(join(tmpdir(), 'ok-supersede-'));
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
+ writeFileSync(
+ join(dir, 'gh'),
+ '#!/usr/bin/env bash\nif [[ -n "$STUB_ERR" ]]; then echo "$STUB_ERR" >&2; exit 1; fi\necho "$STUB_LATEST"\n',
+ { mode: 0o755 },
+ );
+ const run = (env) =>
+ spawnSync('bash', ['-c', step.run], {
+ env: {
+ PATH: `${dir}:${process.env.PATH}`,
+ RUNNER_TEMP: dir,
+ GITHUB_REPOSITORY: 'inkeep/open-knowledge',
+ GITHUB_RUN_ATTEMPT: '2',
+ RELEASE_TAG: 'v0.83.0',
+ STUB_LATEST: '',
+ STUB_ERR: '',
+ ...env,
+ },
+ encoding: 'utf8',
+ });
+
+ test('only guards re-runs that can stamp Latest', () => {
+ expect(step.if).toContain("github.run_attempt != '1'");
+ expect(step.if).toContain("github.event_name != 'workflow_dispatch'");
+ });
+
+ test.each([
+ ['the tag is the latest stable', { STUB_LATEST: 'v0.83.0' }, 0],
+ ['an older stable is latest', { STUB_LATEST: 'v0.82.9' }, 0],
+ ['no stable has shipped', { STUB_ERR: 'gh: Not Found (HTTP 404)' }, 0],
+ ['a newer stable shipped', { STUB_LATEST: 'v0.83.1' }, 1],
+ ['a newer minor shipped', { STUB_LATEST: 'v0.100.0' }, 1],
+ ['the lookup failed', { STUB_ERR: 'gh: Server Error (HTTP 502)' }, 1],
+ ])('exits %#: %s', (_label, env, status) => {
+ expect(run(env).status).toBe(status);
+ });
+});
diff --git a/.github/workflows/desktop-release-auto-retry.yml b/.github/workflows/desktop-release-auto-retry.yml
new file mode 100644
index 000000000..a57cda692
--- /dev/null
+++ b/.github/workflows/desktop-release-auto-retry.yml
@@ -0,0 +1,59 @@
+name: Desktop release auto-retry
+
+on:
+ workflow_run:
+ workflows: [Desktop release]
+ types: [completed]
+
+permissions: {}
+
+jobs:
+ rerun:
+ if: >-
+ github.event.workflow_run.conclusion != 'success' &&
+ github.event.workflow_run.run_attempt == 1 &&
+ github.event.workflow_run.path == '.github/workflows/desktop-release.yml' &&
+ github.event.workflow_run.head_repository.full_name == github.repository &&
+ contains(fromJSON('["repository_dispatch", "workflow_dispatch"]'), github.event.workflow_run.event)
+ runs-on: ubuntu-latest
+ cache-mode: none
+ timeout-minutes: 5
+ permissions:
+ actions: write
+ defaults:
+ run:
+ shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_ID: ${{ github.event.workflow_run.id }}
+ steps:
+ - name: Re-run the failed release jobs once
+ run: |
+ set -euo pipefail
+ handoff="$(timeout 60 gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?name=release-auto-retry" --jq '.total_count')"
+ if [[ "$handoff" == "0" ]]; then
+ echo "Run ${RUN_ID} handed over no automatic re-run."
+ exit 0
+ fi
+ echo "Re-running the failed jobs of run ${RUN_ID}."
+ timeout 60 gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun-failed-jobs"
+
+ - name: Alert when the automatic re-run could not start
+ if: failure() || cancelled()
+ env:
+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
+ SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
+ run: |
+ set -uo pipefail
+ TEXT="🚨 RELEASE BLOCKED: the automatic re-run of ${RUN_URL} could not start — nothing shipped. Re-run its failed jobs by hand: gh run rerun ${RUN_ID} --failed -R ${GITHUB_REPOSITORY}"
+ echo "::error::${TEXT}"
+ webhook="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"
+ if [[ -z "$webhook" ]]; then
+ echo "::notice::Slack webhook not set — skipping the auto-retry alert."
+ exit 0
+ fi
+ if ! curl -sS --fail --max-time 30 -X POST -H 'Content-type: application/json' \
+ --data "$(jq -nc --arg t "$TEXT" '{text: $t}')" "$webhook"; then
+ echo "::warning::The auto-retry alert failed to POST; the release is still blocked."
+ fi
diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml
index 6cabc88af..1cd74a780 100644
--- a/.github/workflows/desktop-release.yml
+++ b/.github/workflows/desktop-release.yml
@@ -1813,6 +1813,27 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Refuse to promote a superseded stable on a re-run
+ if: success() && needs.prepare.outputs.channel == 'latest' && github.run_attempt != '1' && github.event_name != 'workflow_dispatch'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ set -euo pipefail
+ if ! LATEST="$(gh api "repos/${GITHUB_REPOSITORY}/releases/latest" --jq .tag_name 2>"$RUNNER_TEMP/latest.err")"; then
+ if grep -q 'HTTP 404' "$RUNNER_TEMP/latest.err"; then
+ LATEST=""
+ else
+ cat "$RUNNER_TEMP/latest.err" >&2
+ echo "::error::Could not read the latest stable release, so ${RELEASE_TAG} is not promoted on re-run attempt ${GITHUB_RUN_ATTEMPT}."
+ exit 1
+ fi
+ fi
+ if [[ -n "$LATEST" && "$LATEST" != "$RELEASE_TAG" ]] \
+ && [[ "$(printf '%s\n%s\n' "$RELEASE_TAG" "$LATEST" | sort -V | tail -n1)" == "$LATEST" ]]; then
+ echo "::error::Refusing to promote ${RELEASE_TAG} on re-run attempt ${GITHUB_RUN_ATTEMPT}: the newer stable ${LATEST} has already shipped. Cut a new release from main instead."
+ exit 1
+ fi
+
# Marker-delimited (`<!-- ok-downloads:start/end -->`) so re-renders are
# idempotent and the rest of the body — the changelog and the beta
# cadence's hidden consumed-set marker, which compute-next-beta.mjs
@@ -2119,8 +2140,9 @@ jobs:
cache-mode: none
timeout-minutes: 10
permissions:
- # Reads scripts from the checkout, posts to a webhook — nothing more.
contents: read
+ actions: read
+ checks: read
defaults:
run:
shell: bash
@@ -2131,6 +2153,28 @@ jobs:
# $GITHUB_SHA — CI tooling, not release content (see finalize's header).
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Decide on one automatic re-run
+ id: retry
+ if: needs.prepare.outputs.channel == 'latest'
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ SMOKE_VERDICT: ${{ needs.build-macos.outputs.smoke_verdict }}
+ DECISION_PATH: ${{ runner.temp }}/release-auto-retry/decision.json
+ REQUIRED_PLATFORMS: ${{ needs.prepare.outputs.required }}
+ run: node .github/scripts/decide-release-retry.mjs
+
+ - name: Hand the re-run to the follower workflow
+ id: handoff
+ if: steps.retry.outputs.action == 'retry'
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: release-auto-retry
+ path: ${{ runner.temp }}/release-auto-retry/decision.json
+ retention-days: 1
+ if-no-files-found: error
+
# FR5c — page the release channel when the gate refuses a stable
# release. Without this a blocked release is silent: the Release simply
# never appears and nobody learns why until someone goes looking. The
@@ -2151,8 +2195,12 @@ jobs:
# .github/scripts/build-smoke-alert-payload.mjs. Ratcheted by
# build-smoke-alert-payload.test.mjs so this cannot drift back.
- name: Alert on a blocked release (FR5c)
- if: needs.prepare.outputs.channel == 'latest'
+ if: needs.prepare.outputs.channel == 'latest' && (steps.retry.outputs.action != 'retry' || steps.handoff.outcome != 'success')
env:
+ RETRY_OUTCOME: ${{ steps.retry.outcome }}
+ RETRY_ACTION: ${{ steps.retry.outputs.action }}
+ RETRY_REASON: ${{ steps.retry.outputs.reason }}
+ HANDOFF_OUTCOME: ${{ steps.handoff.outcome }}
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SMOKE_VERDICT: ${{ needs.build-macos.outputs.smoke_verdict }}
@@ -2194,13 +2242,13 @@ jobs:
is_required() { case ",${REQUIRED}," in *",$1,"*) return 0 ;; *) return 1 ;; esac; }
blocked=()
[[ "${RESULT_PREPARE}" == "failure" ]] && blocked+=("release preparation")
- if [[ "${RESULT_MACOS}" == "failure" ]] && is_required mac; then
+ if [[ "${RESULT_MACOS}" == "failure" || "${RESULT_MACOS}" == "cancelled" ]] && is_required mac; then
blocked+=("macOS packaging")
fi
- if [[ "${RESULT_WINDOWS}" == "failure" ]] && is_required windows; then
+ if [[ "${RESULT_WINDOWS}" == "failure" || "${RESULT_WINDOWS}" == "cancelled" ]] && is_required windows; then
blocked+=("Windows packaging")
fi
- if [[ "${RESULT_LINUX}" == "failure" ]] && is_required linux; then
+ if [[ "${RESULT_LINUX}" == "failure" || "${RESULT_LINUX}" == "cancelled" ]] && is_required linux; then
blocked+=("Linux packaging")
fi
[[ "${RESULT_PUBLISH}" == "failure" ]] && blocked+=("asset publication")
@@ -2214,6 +2262,13 @@ jobs:
BLOCKED_BY="${BLOCKED_BY:+${BLOCKED_BY}, }${b}"
done
REASON="${SMOKE_REASON:-the release pipeline failed before completing — see the run}"
+ if [[ "${RETRY_OUTCOME:-}" == "failure" ]]; then
+ REASON="${REASON}. Not re-run automatically: the re-run decision failed, see the run"
+ elif [[ "${RETRY_ACTION:-}" == "retry" && "${HANDOFF_OUTCOME:-}" != "success" ]]; then
+ REASON="${REASON}. Not re-run automatically: handing the re-run to the follower workflow failed"
+ elif [[ -n "${RETRY_REASON:-}" ]]; then
+ REASON="${REASON}. Not re-run automatically: ${RETRY_REASON}"
+ fi
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
post() {
local webhook="$1" label="$2"