Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): give the bug lane a flake retry, page releases (PRD-7817) (#3238)

Andrew Mikofalvy3b3c23deb77a
diff --git a/.github/scripts/build-smoke-alert-payload.test.mjs b/.github/scripts/build-smoke-alert-payload.test.mjs
index d2ab3e53a..ff2d4d500 100644
--- a/.github/scripts/build-smoke-alert-payload.test.mjs
+++ b/.github/scripts/build-smoke-alert-payload.test.mjs
@@ -187,8 +187,9 @@ describe('workflow wiring', () => {
expect(step).toMatch(/if \[\[ -z "\$webhook" \]\]; then[\s\S]{0,200}?return 0/);
// A dead webhook warns; it must not mask the smoke failure that caused it.
expect(step).toContain('::warning::${label} smoke alert failed to POST');
- // The channel is actually driven.
- expect(step).toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');
+ // The channel is actually driven, and through the releases-first
+ // resolution rather than straight at the shared secret.
+ expect(step).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
});
test('a blocked release never pages Discord', () => {
@@ -200,7 +201,7 @@ describe('workflow wiring', () => {
test('the annotation is emitted in addition to the page, not instead of it', () => {
const step = alertStep();
- const slackAt = step.indexOf('post "${SLACK_WEBHOOK_URL:-}" Slack');
+ const slackAt = step.indexOf('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
const annotationAt = step.indexOf('::error::RELEASE BLOCKED');
expect(slackAt).toBeGreaterThan(-1);
expect(annotationAt).toBeGreaterThan(slackAt);
diff --git a/.github/scripts/release-cascade-shape.test.mjs b/.github/scripts/release-cascade-shape.test.mjs
index 1bee36a92..adbc6e829 100644
--- a/.github/scripts/release-cascade-shape.test.mjs
+++ b/.github/scripts/release-cascade-shape.test.mjs
@@ -24,6 +24,8 @@ const read = (name) => readFileSync(join(WORKFLOWS, name), 'utf8');
const desktopRelease = read('desktop-release.yml');
const promoteStable = read('promote-stable.yml');
const releaseYml = read('release.yml');
+const bugLane = read('bug-lane.yml');
+const selectBeta = read('select-beta-to-promote.yml');
/**
* Ordered step names across the workflow, in FILE order. Under the fan-out
@@ -378,7 +380,7 @@ describe('a forced smoke failure pages Slack, not just an annotation', () => {
desktopRelease.indexOf('- name: Alert on a blocked release'),
desktopRelease.indexOf('- name: Warn on stuck draft'),
);
- expect(step).toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');
+ expect(step).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
expect(step).toContain('::error::RELEASE BLOCKED');
});
});
@@ -452,7 +454,68 @@ describe('the stable gate does not touch the beta cadence', () => {
});
});
-describe('release notes and pipeline ops route to independent webhooks', () => {
+describe('the bug lane verifies the synthetic tree at the same bar as main', () => {
+ const verify = bugLane.slice(
+ bugLane.indexOf('- name: Verify the synthetic tree'),
+ bugLane.indexOf('- name: Dispatch the point release'),
+ );
+
+ test('a red tier gets one retry before the tick is refused', () => {
+ // Single-shot here holds a candidate to a STRICTER bar than the tier that
+ // gates main, which retries each package once and treats a retry-pass as
+ // green. These tiers spawn real processes (an orphaned CLI reaped on host
+ // death), so one flake under runner contention refused a sound fix on
+ // 2026-08-04 and the page blamed the fix for it.
+ const runs = [...verify.matchAll(/turbo run typecheck test/g)];
+ expect(
+ runs.length,
+ 'verify must invoke the tiers twice: once, then one flake retry',
+ ).toBe(2);
+ expect(verify).toContain('elif pnpm exec turbo run typecheck test');
+ });
+
+ test('the first attempt runs to completion so the retry stays incremental', () => {
+ // Load-bearing against this job's cancel window, not a style choice.
+ // turbo defaults to `--continue=never`, which cancels every in-flight and
+ // unstarted task on the first failure — none of those cache, so the retry
+ // re-runs them and a late failure costs close to two full passes. With
+ // `cancel-in-progress: true` a tick that outruns the next one is killed
+ // before it can page, losing the refusal entirely.
+ // Scoped to the invocation LINES, not the step text: the comment above
+ // them names both flags to explain the choice, and a ratchet that bans
+ // naming what it rules out just gets worked around.
+ const invocations = verify
+ .split('\n')
+ .filter((l) => l.includes('pnpm exec turbo run typecheck test'));
+ expect(invocations).toHaveLength(2);
+ const [first, retry] = invocations;
+ expect(first).toContain('--continue');
+ // `--force` on either would throw away exactly the cached passes that
+ // make the retry cheap.
+ expect(first).not.toContain('--force');
+ expect(retry).not.toContain('--force');
+ });
+
+ test('only a second consecutive failure mints verdict=fail', () => {
+ const failAt = verify.indexOf('verdict=fail');
+ const elifAt = verify.indexOf('elif pnpm exec turbo run typecheck test');
+ expect(elifAt).toBeGreaterThan(-1);
+ // The install-failure guard mints its own verdict=fail earlier; the one
+ // that matters here is the tier verdict, which must sit after the retry.
+ expect(verify.lastIndexOf('verdict=fail')).toBeGreaterThan(elifAt);
+ expect(failAt).toBeGreaterThan(-1);
+ });
+
+ test('the refusal page does not claim a cause it has not established', () => {
+ // The prior text asserted "the fix passes on main but not on the stable it
+ // would ship against" off a single red run, sending operators hunting for
+ // an incompatibility that was really a flake.
+ const page = bugLane.slice(bugLane.indexOf('- name: Page on a refusal'));
+ expect(page).not.toContain('the fix passes on main but not on the stable');
+ });
+});
+
+describe('every release-pipeline post prefers the releases webhook', () => {
const announce = desktopRelease.slice(
desktopRelease.indexOf('- name: Announce stable release to Slack'),
desktopRelease.indexOf('- name: Announce stable release to Discord'),
@@ -482,13 +545,66 @@ describe('release notes and pipeline ops route to independent webhooks', () => {
expect(announce).toContain('if [[ -z "$WEBHOOK_URL" ]]; then');
});
- test('the smoke alarm keeps the shared webhook, so moving the notes cannot drag it', () => {
- // The alarm pages pipeline ops, not release notes; it must keep paging
- // wherever it pages today even after the notes move channel.
+ test('the blocked-release alarm resolves the same way the announcement does', () => {
+ // The alarm is the negative of the announcement — the release that did NOT
+ // ship — so it reads as release traffic and belongs with the notes. What
+ // must never come back is a bare "$SLACK_WEBHOOK_URL" post: that still
+ // exits 0 while the page silently reappears in the product channel.
const alert = desktopRelease.slice(
desktopRelease.indexOf('- name: Alert on a blocked release'),
);
- expect(alert).toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');
- expect(alert).not.toContain('SLACK_RELEASES_WEBHOOK_URL');
+ expect(alert).toContain(
+ 'SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}',
+ );
+ expect(alert).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
+ expect(alert).not.toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');
+ });
+
+ // The remaining four moved posts. Ratcheting only the blocked-release alert
+ // would leave the stated invariant — no step posts straight at the shared
+ // secret — unenforced for most of the steps that moved, and this is the
+ // regression class that exits 0 while the page reappears in the product
+ // channel, so it is invisible without a test.
+ const RESOLVED = 'WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"';
+ const stepAfter = (source, name, next) =>
+ source.slice(
+ source.indexOf(`- name: ${name}`),
+ next === undefined ? undefined : source.indexOf(`- name: ${next}`),
+ );
+
+ for (const { label, step } of [
+ {
+ label: "the bug lane's refusal page",
+ step: () => stepAfter(bugLane, 'Page on a refusal'),
+ },
+ {
+ label: "the bug lane's partial-drop notice",
+ step: () => stepAfter(bugLane, 'Notify on a partial drop', 'Page on a refusal'),
+ },
+ {
+ label: 'the fast-tier refusal',
+ step: () => stepAfter(selectBeta, 'Record a fast-tier refusal'),
+ },
+ ]) {
+ test(`${label} resolves the releases webhook first`, () => {
+ const s = step();
+ expect(s).toContain('SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}');
+ expect(s).toContain(RESOLVED);
+ expect(s).toContain('"$WEBHOOK_URL"');
+ // The bare forms this replaced. Either one reaching the curl again puts
+ // the page back in the product channel with nothing failing.
+ expect(s).not.toContain('--data "$payload" "$SLACK_WEBHOOK_URL"');
+ expect(s).not.toContain('if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then');
+ });
+ }
+
+ test('the aggregate smoke alarm resolves the releases webhook first', () => {
+ // This one posts through a `post` helper rather than a bare curl, so it
+ // carries the compound expansion at the call site instead of a WEBHOOK_URL
+ // assignment.
+ const alarm = stepAfter(selectBeta, 'Page the release channel');
+ expect(alarm).toContain('SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}');
+ expect(alarm).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
+ expect(alarm).not.toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');
});
});
diff --git a/.github/workflows/bug-lane.yml b/.github/workflows/bug-lane.yml
index 09d84b21c..afc640fdb 100644
--- a/.github/workflows/bug-lane.yml
+++ b/.github/workflows/bug-lane.yml
@@ -174,11 +174,33 @@ jobs:
echo "::warning::bug-lane: pnpm install --frozen-lockfile failed on the synthetic tree (${STABLE} + ${SURVIVING_REFS}); treated as a red verification."
exit 0
fi
- if pnpm exec turbo run typecheck test --output-logs=errors-only; then
+ # ONE flake retry, matching the stance the PR tier already applies to
+ # this same suite. Single-shot would hold the synthetic tree to a
+ # STRICTER bar than the tree that gates main, and these tiers include
+ # process-lifecycle probes that spawn a real CLI, SIGKILL its parent
+ # and poll for the grandchild to exit — exactly the class that flakes
+ # under runner contention. Unretried, one such flake refuses a fix
+ # that is fine, and the refusal reads as a real incompatibility.
+ #
+ # `--continue` on the FIRST attempt is what keeps the retry cheap,
+ # and it is load-bearing against this job's own cancel window. turbo
+ # defaults to `--continue=never`: the first failure cancels every
+ # in-flight and unstarted task, so none of those cache and the retry
+ # re-runs them too — a failure late in the tiers would cost close to
+ # two full passes, and with `cancel-in-progress` a tick that outruns
+ # the next one is killed before it can page, which is the one outcome
+ # the install guard above exists to prevent. Running the first
+ # attempt to completion caches every task that genuinely passed, so
+ # the retry re-executes only what actually failed. `--force` is
+ # omitted for the same reason: it would discard those cached passes.
+ if pnpm exec turbo run typecheck test --continue --output-logs=errors-only; then
echo "verdict=pass" >> "$GITHUB_OUTPUT"
+ elif pnpm exec turbo run typecheck test --output-logs=errors-only; then
+ echo "verdict=pass" >> "$GITHUB_OUTPUT"
+ echo "::warning::bug-lane: fast tiers passed on retry for ${STABLE} + ${SURVIVING_REFS} — the first attempt failed (flake-class); see its log before trusting this tick."
else
echo "verdict=fail" >> "$GITHUB_OUTPUT"
- echo "::warning::bug-lane: fast tiers FAILED on ${STABLE} + ${SURVIVING_REFS} — the fix passes on main but not on the stable it would ship against. It stays on its cycle's path."
+ echo "::warning::bug-lane: fast tiers failed TWICE on ${STABLE} + ${SURVIVING_REFS} — not flake-class, so the fix likely assumes something the stable does not have. It stays on its cycle's path."
fi
- name: Dispatch the point release (or log what would ship)
@@ -225,25 +247,28 @@ jobs:
# lane's most common non-trivial outcome and must not be log-only.
if: env.BUG_LANE_ARMED == 'true' && steps.verify.outputs.verdict == 'pass' && steps.verify.outputs.dropped_refs != ''
env:
+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
DROPPED_REFS: ${{ steps.verify.outputs.dropped_refs }}
SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}
STABLE: ${{ steps.evaluate.outputs.stable }}
run: |
set -uo pipefail
- if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then
- echo "::notice::SLACK_WEBHOOK_URL not set — drop notification skipped."
+ WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"
+ if [[ -z "$WEBHOOK_URL" ]]; then
+ echo "::notice::No Slack webhook secret set — drop notification skipped."
exit 0
fi
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
TEXT="⚠️ Bug lane dropped ${DROPPED_REFS} (cherry-pick conflict with ${STABLE}); they ride their cycle's 24h path — investigate whether any needs a manual point release. Dispatched the self-contained subset ${SURVIVING_REFS}. ${RUN_URL}"
payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"
- curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" \
+ curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL" \
|| echo "::warning::bug-lane drop notification failed to POST — the dispatch itself is unaffected."
- name: Page on a refusal (armed only)
if: env.BUG_LANE_ARMED == 'true' && (steps.verify.outputs.verdict == 'conflict' || steps.verify.outputs.verdict == 'fail')
env:
+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
FIX_REFS: ${{ steps.evaluate.outputs.fix_refs }}
SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}
@@ -251,8 +276,9 @@ jobs:
VERDICT: ${{ steps.verify.outputs.verdict }}
run: |
set -uo pipefail
- if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then
- echo "::notice::SLACK_WEBHOOK_URL not set — refusal page skipped."
+ WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"
+ if [[ -z "$WEBHOOK_URL" ]]; then
+ echo "::notice::No Slack webhook secret set — refusal page skipped."
exit 0
fi
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
@@ -262,12 +288,12 @@ jobs:
if [[ "$VERDICT" == "conflict" ]]; then
TEXT="⛔ Bug lane: no qualifying fix is self-contained over the stable — all of ${FIX_REFS} conflict. They ride their cycle's 24h path; investigate whether any needs a manual point release. ${RUN_URL}"
else
- TEXT="⛔ Bug lane: verification failed on the synthetic tree for ${SURVIVING_REFS} (red tiers or install failure)."
+ TEXT="⛔ Bug lane: verification failed on the synthetic tree for ${SURVIVING_REFS} — the fast tiers went red on BOTH attempts, or the install failed. Either way this is past flake-class: treat it as a real conflict with the stable, not a retryable blip."
if [[ -n "${DROPPED_REFS}" ]]; then
TEXT="${TEXT} Also dropped for conflict: ${DROPPED_REFS}."
fi
TEXT="${TEXT} The fixes ride their cycle's 24h path. ${RUN_URL}"
fi
payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"
- curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" \
+ curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL" \
|| echo "::warning::bug-lane refusal page failed to POST — the refusal itself still holds."
diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml
index d951f4268..c14cdb26c 100644
--- a/.github/workflows/desktop-release.yml
+++ b/.github/workflows/desktop-release.yml
@@ -1634,11 +1634,12 @@ jobs:
# A Slack incoming webhook is bound to one channel when it is installed,
# and the payload below carries no `channel` field, so the destination is
# decided entirely by which webhook URL this step posts to.
- # SLACK_RELEASES_WEBHOOK_URL puts release notes in their own channel.
- # SLACK_WEBHOOK_URL is the fallback, and is shared with the smoke alarms
- # and refusal pages — those are pipeline ops rather than release notes,
- # so they keep their own destination and must not be moved by rotating
- # the shared secret. With neither set (a fork, or before either is
+ # SLACK_RELEASES_WEBHOOK_URL puts release notes in their own channel, and
+ # every release-pipeline page now prefers it too — the smoke alarms, the
+ # promotion pages, and the bug lane's refusal and drop notices all read
+ # as release traffic to the humans watching, so splitting them across two
+ # channels only made the product channel noisy. SLACK_WEBHOOK_URL stays
+ # the fallback everywhere. With neither set (a fork, or before either is
# provisioned) the step no-ops cleanly; a Slack delivery error is
# downgraded to a ::warning:: — the release is "done" once published, so
# the announcement is strictly best-effort and must never fail an
@@ -1788,8 +1789,11 @@ jobs:
# keeps the STUCK-DRAFT warning below; paging the release channel for
# every beta hiccup would train readers to ignore the alert.
#
- # No new secret: SLACK_WEBHOOK_URL already exists and is consumed by the
- # announcement step with the same unset -> ::notice:: -> skip tolerance.
+ # Same destination as the release announcement: this alert is the
+ # negative of that message (the release that did NOT ship), so it belongs
+ # with the notes rather than in the product channel. SLACK_WEBHOOK_URL
+ # remains the fallback, with the same unset -> ::notice:: -> skip
+ # tolerance.
#
# Slack only, deliberately — rationale lives in the module docstring of
# .github/scripts/build-smoke-alert-payload.mjs. Ratcheted by
@@ -1797,6 +1801,7 @@ jobs:
- name: Alert on a blocked release (FR5c)
if: needs.prepare.outputs.channel == 'latest'
env:
+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SMOKE_VERDICT: ${{ needs.build-macos.outputs.smoke_verdict }}
SMOKE_REASON: ${{ needs.build-macos.outputs.smoke_reason }}
@@ -1842,7 +1847,7 @@ jobs:
fi
}
- post "${SLACK_WEBHOOK_URL:-}" Slack
+ post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack
# The annotation is emitted IN ADDITION to the page, so the run page
# tells the same story as the channel.
echo "::error::RELEASE BLOCKED: ${RELEASE_TAG} pipeline ${VERDICT} — ${REASON}. The Release stays a draft and npm latest is untouched."
diff --git a/.github/workflows/select-beta-to-promote.yml b/.github/workflows/select-beta-to-promote.yml
index 9b1d75cba..d1802c131 100644
--- a/.github/workflows/select-beta-to-promote.yml
+++ b/.github/workflows/select-beta-to-promote.yml
@@ -390,6 +390,7 @@ jobs:
env:
CANDIDATE: ${{ needs.evaluate.outputs.fast_tier_candidate }}
VERDICT: ${{ steps.smoke.outputs.verdict }}
+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
run: |
set -uo pipefail
@@ -399,14 +400,15 @@ jobs:
# nobody is watching, which makes the gate decorative. Best-effort:
# a failed POST is a warning, never a job failure — the refusal
# itself already protected the release.
- if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then
- echo "::notice::SLACK_WEBHOOK_URL not set — refusal page skipped."
+ WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"
+ if [[ -z "$WEBHOOK_URL" ]]; then
+ echo "::notice::No Slack webhook secret set — refusal page skipped."
exit 0
fi
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
TEXT="⛔ Fast-tier DMG smoke REFUSED for ${CANDIDATE} (verdict=${VERDICT:-unknown}). The beta stays on the 24h soak; investigate the smoke before it promotes. ${RUN_URL}"
payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"
- if curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL"; then
+ if curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL"; then
echo "Posted the fast-tier refusal page."
else
echo "::warning::Fast-tier refusal page failed to POST — the refusal itself still holds."
@@ -462,6 +464,7 @@ jobs:
- name: Page the release channel
if: steps.alarm.outputs.alarm == 'true'
env:
+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
REASONS: ${{ steps.alarm.outputs.reasons }}
run: |
@@ -483,5 +486,5 @@ jobs:
echo "::warning::${label} aggregate alarm failed to POST — the alarm condition still holds."
fi
}
- post "${SLACK_WEBHOOK_URL:-}" Slack
+ post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack
echo "::warning::DMG smoke gate alarm: ${REASONS}"