Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): give the bug lane a flake retry, page releases (PRD-7817) (#3238)
Andrew Mikofalvy3b3c23deb77a
diff --git a/.github/scripts/build-smoke-alert-payload.test.mjs b/.github/scripts/build-smoke-alert-payload.test.mjsindex d2ab3e53a..ff2d4d500 100644--- a/.github/scripts/build-smoke-alert-payload.test.mjs+++ b/.github/scripts/build-smoke-alert-payload.test.mjs@@ -187,8 +187,9 @@ describe('workflow wiring', () => {expect(step).toMatch(/if \[\[ -z "\$webhook" \]\]; then[\s\S]{0,200}?return 0/);// A dead webhook warns; it must not mask the smoke failure that caused it.expect(step).toContain('::warning::${label} smoke alert failed to POST');- // The channel is actually driven.- expect(step).toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');+ // The channel is actually driven, and through the releases-first+ // resolution rather than straight at the shared secret.+ expect(step).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');});test('a blocked release never pages Discord', () => {@@ -200,7 +201,7 @@ describe('workflow wiring', () => {test('the annotation is emitted in addition to the page, not instead of it', () => {const step = alertStep();- const slackAt = step.indexOf('post "${SLACK_WEBHOOK_URL:-}" Slack');+ const slackAt = step.indexOf('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');const annotationAt = step.indexOf('::error::RELEASE BLOCKED');expect(slackAt).toBeGreaterThan(-1);expect(annotationAt).toBeGreaterThan(slackAt);diff --git a/.github/scripts/release-cascade-shape.test.mjs b/.github/scripts/release-cascade-shape.test.mjsindex 1bee36a92..adbc6e829 100644--- a/.github/scripts/release-cascade-shape.test.mjs+++ b/.github/scripts/release-cascade-shape.test.mjs@@ -24,6 +24,8 @@ const read = (name) => readFileSync(join(WORKFLOWS, name), 'utf8');const desktopRelease = read('desktop-release.yml');const promoteStable = read('promote-stable.yml');const releaseYml = read('release.yml');+const bugLane = read('bug-lane.yml');+const selectBeta = read('select-beta-to-promote.yml');/*** Ordered step names across the workflow, in FILE order. Under the fan-out@@ -378,7 +380,7 @@ describe('a forced smoke failure pages Slack, not just an annotation', () => {desktopRelease.indexOf('- name: Alert on a blocked release'),desktopRelease.indexOf('- name: Warn on stuck draft'),);- expect(step).toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');+ expect(step).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');expect(step).toContain('::error::RELEASE BLOCKED');});});@@ -452,7 +454,68 @@ describe('the stable gate does not touch the beta cadence', () => {});});-describe('release notes and pipeline ops route to independent webhooks', () => {+describe('the bug lane verifies the synthetic tree at the same bar as main', () => {+ const verify = bugLane.slice(+ bugLane.indexOf('- name: Verify the synthetic tree'),+ bugLane.indexOf('- name: Dispatch the point release'),+ );++ test('a red tier gets one retry before the tick is refused', () => {+ // Single-shot here holds a candidate to a STRICTER bar than the tier that+ // gates main, which retries each package once and treats a retry-pass as+ // green. These tiers spawn real processes (an orphaned CLI reaped on host+ // death), so one flake under runner contention refused a sound fix on+ // 2026-08-04 and the page blamed the fix for it.+ const runs = [...verify.matchAll(/turbo run typecheck test/g)];+ expect(+ runs.length,+ 'verify must invoke the tiers twice: once, then one flake retry',+ ).toBe(2);+ expect(verify).toContain('elif pnpm exec turbo run typecheck test');+ });++ test('the first attempt runs to completion so the retry stays incremental', () => {+ // Load-bearing against this job's cancel window, not a style choice.+ // turbo defaults to `--continue=never`, which cancels every in-flight and+ // unstarted task on the first failure — none of those cache, so the retry+ // re-runs them and a late failure costs close to two full passes. With+ // `cancel-in-progress: true` a tick that outruns the next one is killed+ // before it can page, losing the refusal entirely.+ // Scoped to the invocation LINES, not the step text: the comment above+ // them names both flags to explain the choice, and a ratchet that bans+ // naming what it rules out just gets worked around.+ const invocations = verify+ .split('\n')+ .filter((l) => l.includes('pnpm exec turbo run typecheck test'));+ expect(invocations).toHaveLength(2);+ const [first, retry] = invocations;+ expect(first).toContain('--continue');+ // `--force` on either would throw away exactly the cached passes that+ // make the retry cheap.+ expect(first).not.toContain('--force');+ expect(retry).not.toContain('--force');+ });++ test('only a second consecutive failure mints verdict=fail', () => {+ const failAt = verify.indexOf('verdict=fail');+ const elifAt = verify.indexOf('elif pnpm exec turbo run typecheck test');+ expect(elifAt).toBeGreaterThan(-1);+ // The install-failure guard mints its own verdict=fail earlier; the one+ // that matters here is the tier verdict, which must sit after the retry.+ expect(verify.lastIndexOf('verdict=fail')).toBeGreaterThan(elifAt);+ expect(failAt).toBeGreaterThan(-1);+ });++ test('the refusal page does not claim a cause it has not established', () => {+ // The prior text asserted "the fix passes on main but not on the stable it+ // would ship against" off a single red run, sending operators hunting for+ // an incompatibility that was really a flake.+ const page = bugLane.slice(bugLane.indexOf('- name: Page on a refusal'));+ expect(page).not.toContain('the fix passes on main but not on the stable');+ });+});++describe('every release-pipeline post prefers the releases webhook', () => {const announce = desktopRelease.slice(desktopRelease.indexOf('- name: Announce stable release to Slack'),desktopRelease.indexOf('- name: Announce stable release to Discord'),@@ -482,13 +545,66 @@ describe('release notes and pipeline ops route to independent webhooks', () => {expect(announce).toContain('if [[ -z "$WEBHOOK_URL" ]]; then');});- test('the smoke alarm keeps the shared webhook, so moving the notes cannot drag it', () => {- // The alarm pages pipeline ops, not release notes; it must keep paging- // wherever it pages today even after the notes move channel.+ test('the blocked-release alarm resolves the same way the announcement does', () => {+ // The alarm is the negative of the announcement — the release that did NOT+ // ship — so it reads as release traffic and belongs with the notes. What+ // must never come back is a bare "$SLACK_WEBHOOK_URL" post: that still+ // exits 0 while the page silently reappears in the product channel.const alert = desktopRelease.slice(desktopRelease.indexOf('- name: Alert on a blocked release'),);- expect(alert).toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');- expect(alert).not.toContain('SLACK_RELEASES_WEBHOOK_URL');+ expect(alert).toContain(+ 'SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}',+ );+ expect(alert).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');+ expect(alert).not.toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');+ });++ // The remaining four moved posts. Ratcheting only the blocked-release alert+ // would leave the stated invariant — no step posts straight at the shared+ // secret — unenforced for most of the steps that moved, and this is the+ // regression class that exits 0 while the page reappears in the product+ // channel, so it is invisible without a test.+ const RESOLVED = 'WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"';+ const stepAfter = (source, name, next) =>+ source.slice(+ source.indexOf(`- name: ${name}`),+ next === undefined ? undefined : source.indexOf(`- name: ${next}`),+ );++ for (const { label, step } of [+ {+ label: "the bug lane's refusal page",+ step: () => stepAfter(bugLane, 'Page on a refusal'),+ },+ {+ label: "the bug lane's partial-drop notice",+ step: () => stepAfter(bugLane, 'Notify on a partial drop', 'Page on a refusal'),+ },+ {+ label: 'the fast-tier refusal',+ step: () => stepAfter(selectBeta, 'Record a fast-tier refusal'),+ },+ ]) {+ test(`${label} resolves the releases webhook first`, () => {+ const s = step();+ expect(s).toContain('SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}');+ expect(s).toContain(RESOLVED);+ expect(s).toContain('"$WEBHOOK_URL"');+ // The bare forms this replaced. Either one reaching the curl again puts+ // the page back in the product channel with nothing failing.+ expect(s).not.toContain('--data "$payload" "$SLACK_WEBHOOK_URL"');+ expect(s).not.toContain('if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then');+ });+ }++ test('the aggregate smoke alarm resolves the releases webhook first', () => {+ // This one posts through a `post` helper rather than a bare curl, so it+ // carries the compound expansion at the call site instead of a WEBHOOK_URL+ // assignment.+ const alarm = stepAfter(selectBeta, 'Page the release channel');+ expect(alarm).toContain('SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}');+ expect(alarm).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');+ expect(alarm).not.toContain('post "${SLACK_WEBHOOK_URL:-}" Slack');});});diff --git a/.github/workflows/bug-lane.yml b/.github/workflows/bug-lane.ymlindex 09d84b21c..afc640fdb 100644--- a/.github/workflows/bug-lane.yml+++ b/.github/workflows/bug-lane.yml@@ -174,11 +174,33 @@ jobs:echo "::warning::bug-lane: pnpm install --frozen-lockfile failed on the synthetic tree (${STABLE} + ${SURVIVING_REFS}); treated as a red verification."exit 0fi- if pnpm exec turbo run typecheck test --output-logs=errors-only; then+ # ONE flake retry, matching the stance the PR tier already applies to+ # this same suite. Single-shot would hold the synthetic tree to a+ # STRICTER bar than the tree that gates main, and these tiers include+ # process-lifecycle probes that spawn a real CLI, SIGKILL its parent+ # and poll for the grandchild to exit — exactly the class that flakes+ # under runner contention. Unretried, one such flake refuses a fix+ # that is fine, and the refusal reads as a real incompatibility.+ #+ # `--continue` on the FIRST attempt is what keeps the retry cheap,+ # and it is load-bearing against this job's own cancel window. turbo+ # defaults to `--continue=never`: the first failure cancels every+ # in-flight and unstarted task, so none of those cache and the retry+ # re-runs them too — a failure late in the tiers would cost close to+ # two full passes, and with `cancel-in-progress` a tick that outruns+ # the next one is killed before it can page, which is the one outcome+ # the install guard above exists to prevent. Running the first+ # attempt to completion caches every task that genuinely passed, so+ # the retry re-executes only what actually failed. `--force` is+ # omitted for the same reason: it would discard those cached passes.+ if pnpm exec turbo run typecheck test --continue --output-logs=errors-only; thenecho "verdict=pass" >> "$GITHUB_OUTPUT"+ elif pnpm exec turbo run typecheck test --output-logs=errors-only; then+ echo "verdict=pass" >> "$GITHUB_OUTPUT"+ echo "::warning::bug-lane: fast tiers passed on retry for ${STABLE} + ${SURVIVING_REFS} — the first attempt failed (flake-class); see its log before trusting this tick."elseecho "verdict=fail" >> "$GITHUB_OUTPUT"- echo "::warning::bug-lane: fast tiers FAILED on ${STABLE} + ${SURVIVING_REFS} — the fix passes on main but not on the stable it would ship against. It stays on its cycle's path."+ echo "::warning::bug-lane: fast tiers failed TWICE on ${STABLE} + ${SURVIVING_REFS} — not flake-class, so the fix likely assumes something the stable does not have. It stays on its cycle's path."fi- name: Dispatch the point release (or log what would ship)@@ -225,25 +247,28 @@ jobs:# lane's most common non-trivial outcome and must not be log-only.if: env.BUG_LANE_ARMED == 'true' && steps.verify.outputs.verdict == 'pass' && steps.verify.outputs.dropped_refs != ''env:+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}DROPPED_REFS: ${{ steps.verify.outputs.dropped_refs }}SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}STABLE: ${{ steps.evaluate.outputs.stable }}run: |set -uo pipefail- if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then- echo "::notice::SLACK_WEBHOOK_URL not set — drop notification skipped."+ WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"+ if [[ -z "$WEBHOOK_URL" ]]; then+ echo "::notice::No Slack webhook secret set — drop notification skipped."exit 0fiRUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"TEXT="⚠️ Bug lane dropped ${DROPPED_REFS} (cherry-pick conflict with ${STABLE}); they ride their cycle's 24h path — investigate whether any needs a manual point release. Dispatched the self-contained subset ${SURVIVING_REFS}. ${RUN_URL}"payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"- curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" \+ curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL" \|| echo "::warning::bug-lane drop notification failed to POST — the dispatch itself is unaffected."- name: Page on a refusal (armed only)if: env.BUG_LANE_ARMED == 'true' && (steps.verify.outputs.verdict == 'conflict' || steps.verify.outputs.verdict == 'fail')env:+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}FIX_REFS: ${{ steps.evaluate.outputs.fix_refs }}SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}@@ -251,8 +276,9 @@ jobs:VERDICT: ${{ steps.verify.outputs.verdict }}run: |set -uo pipefail- if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then- echo "::notice::SLACK_WEBHOOK_URL not set — refusal page skipped."+ WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"+ if [[ -z "$WEBHOOK_URL" ]]; then+ echo "::notice::No Slack webhook secret set — refusal page skipped."exit 0fiRUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"@@ -262,12 +288,12 @@ jobs:if [[ "$VERDICT" == "conflict" ]]; thenTEXT="⛔ Bug lane: no qualifying fix is self-contained over the stable — all of ${FIX_REFS} conflict. They ride their cycle's 24h path; investigate whether any needs a manual point release. ${RUN_URL}"else- TEXT="⛔ Bug lane: verification failed on the synthetic tree for ${SURVIVING_REFS} (red tiers or install failure)."+ TEXT="⛔ Bug lane: verification failed on the synthetic tree for ${SURVIVING_REFS} — the fast tiers went red on BOTH attempts, or the install failed. Either way this is past flake-class: treat it as a real conflict with the stable, not a retryable blip."if [[ -n "${DROPPED_REFS}" ]]; thenTEXT="${TEXT} Also dropped for conflict: ${DROPPED_REFS}."fiTEXT="${TEXT} The fixes ride their cycle's 24h path. ${RUN_URL}"fipayload="$(jq -nc --arg t "$TEXT" '{text: $t}')"- curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL" \+ curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL" \|| echo "::warning::bug-lane refusal page failed to POST — the refusal itself still holds."diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.ymlindex d951f4268..c14cdb26c 100644--- a/.github/workflows/desktop-release.yml+++ b/.github/workflows/desktop-release.yml@@ -1634,11 +1634,12 @@ jobs:# A Slack incoming webhook is bound to one channel when it is installed,# and the payload below carries no `channel` field, so the destination is# decided entirely by which webhook URL this step posts to.- # SLACK_RELEASES_WEBHOOK_URL puts release notes in their own channel.- # SLACK_WEBHOOK_URL is the fallback, and is shared with the smoke alarms- # and refusal pages — those are pipeline ops rather than release notes,- # so they keep their own destination and must not be moved by rotating- # the shared secret. With neither set (a fork, or before either is+ # SLACK_RELEASES_WEBHOOK_URL puts release notes in their own channel, and+ # every release-pipeline page now prefers it too — the smoke alarms, the+ # promotion pages, and the bug lane's refusal and drop notices all read+ # as release traffic to the humans watching, so splitting them across two+ # channels only made the product channel noisy. SLACK_WEBHOOK_URL stays+ # the fallback everywhere. With neither set (a fork, or before either is# provisioned) the step no-ops cleanly; a Slack delivery error is# downgraded to a ::warning:: — the release is "done" once published, so# the announcement is strictly best-effort and must never fail an@@ -1788,8 +1789,11 @@ jobs:# keeps the STUCK-DRAFT warning below; paging the release channel for# every beta hiccup would train readers to ignore the alert.#- # No new secret: SLACK_WEBHOOK_URL already exists and is consumed by the- # announcement step with the same unset -> ::notice:: -> skip tolerance.+ # Same destination as the release announcement: this alert is the+ # negative of that message (the release that did NOT ship), so it belongs+ # with the notes rather than in the product channel. SLACK_WEBHOOK_URL+ # remains the fallback, with the same unset -> ::notice:: -> skip+ # tolerance.## Slack only, deliberately — rationale lives in the module docstring of# .github/scripts/build-smoke-alert-payload.mjs. Ratcheted by@@ -1797,6 +1801,7 @@ jobs:- name: Alert on a blocked release (FR5c)if: needs.prepare.outputs.channel == 'latest'env:+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}SMOKE_VERDICT: ${{ needs.build-macos.outputs.smoke_verdict }}SMOKE_REASON: ${{ needs.build-macos.outputs.smoke_reason }}@@ -1842,7 +1847,7 @@ jobs:fi}- post "${SLACK_WEBHOOK_URL:-}" Slack+ post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack# The annotation is emitted IN ADDITION to the page, so the run page# tells the same story as the channel.echo "::error::RELEASE BLOCKED: ${RELEASE_TAG} pipeline ${VERDICT} — ${REASON}. The Release stays a draft and npm latest is untouched."diff --git a/.github/workflows/select-beta-to-promote.yml b/.github/workflows/select-beta-to-promote.ymlindex 9b1d75cba..d1802c131 100644--- a/.github/workflows/select-beta-to-promote.yml+++ b/.github/workflows/select-beta-to-promote.yml@@ -390,6 +390,7 @@ jobs:env:CANDIDATE: ${{ needs.evaluate.outputs.fast_tier_candidate }}VERDICT: ${{ steps.smoke.outputs.verdict }}+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}run: |set -uo pipefail@@ -399,14 +400,15 @@ jobs:# nobody is watching, which makes the gate decorative. Best-effort:# a failed POST is a warning, never a job failure — the refusal# itself already protected the release.- if [[ -z "${SLACK_WEBHOOK_URL:-}" ]]; then- echo "::notice::SLACK_WEBHOOK_URL not set — refusal page skipped."+ WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"+ if [[ -z "$WEBHOOK_URL" ]]; then+ echo "::notice::No Slack webhook secret set — refusal page skipped."exit 0fiRUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"TEXT="⛔ Fast-tier DMG smoke REFUSED for ${CANDIDATE} (verdict=${VERDICT:-unknown}). The beta stays on the 24h soak; investigate the smoke before it promotes. ${RUN_URL}"payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"- if curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$SLACK_WEBHOOK_URL"; then+ if curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL"; thenecho "Posted the fast-tier refusal page."elseecho "::warning::Fast-tier refusal page failed to POST — the refusal itself still holds."@@ -462,6 +464,7 @@ jobs:- name: Page the release channelif: steps.alarm.outputs.alarm == 'true'env:+ SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}REASONS: ${{ steps.alarm.outputs.reasons }}run: |@@ -483,5 +486,5 @@ jobs:echo "::warning::${label} aggregate alarm failed to POST — the alarm condition still holds."fi}- post "${SLACK_WEBHOOK_URL:-}" Slack+ post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slackecho "::warning::DMG smoke gate alarm: ${REASONS}"