Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): page a bug-lane partial drop once, not once per tick (#3790)

Andrew Mikofalvy131b3c3323d0
diff --git a/.github/scripts/release-cascade-shape.test.mjs b/.github/scripts/release-cascade-shape.test.mjs
index 321ee0b9a..d008a6c7d 100644
--- a/.github/scripts/release-cascade-shape.test.mjs
+++ b/.github/scripts/release-cascade-shape.test.mjs
@@ -25,6 +25,24 @@ const desktopRelease = read('desktop-release.yml');
const promoteStable = read('promote-stable.yml');
const releaseYml = read('release.yml');
const bugLane = read('bug-lane.yml');
+
+/**
+ * One workflow step's body, bounded at the NEXT step.
+ *
+ * Assertions about a step have to be scoped to that step. A whole-file
+ * `toContain` passes on any occurrence, so a string that appears in two steps
+ * lets either one be mutated while the other keeps the test green; a
+ * fixed-width slice runs past a short step into its neighbour and does the
+ * same. Both holes were shipped and caught by mutation testing rather than by
+ * review, which is why this is the only way these files slice a step.
+ */
+const bugLaneStep = (name) => {
+ const start = bugLane.indexOf(`- name: ${name}`);
+ if (start === -1) throw new Error(`bug-lane.yml has no step named ${name}`);
+ const rest = bugLane.slice(start);
+ const end = rest.indexOf('\n - name: ');
+ return end === -1 ? rest : rest.slice(0, end);
+};
const selectBeta = read('select-beta-to-promote.yml');
/**
@@ -531,19 +549,96 @@ describe('the bug lane verifies the synthetic tree at the same bar as main', ()
//
// Step ORDER is not the property: `Record` sits after `Page` either way,
// so an ordering assertion stays green while the guarantee is gone.
- expect(bugLane).toContain('echo "delivered=${delivered}" >> "$GITHUB_OUTPUT"');
- expect(bugLane).toContain('delivered=true');
+ // Scoped to the refusal step. Both strings now also appear in the drop
+ // step added alongside it, so a whole-file assertion would let either
+ // step's copy satisfy the other's test — the sibling-coverage hole that
+ // has bitten every ratchet in this block.
+ const refusalPage = bugLaneStep('Page on a refusal (armed only)');
+ expect(refusalPage).toContain('echo "delivered=${delivered}" >> "$GITHUB_OUTPUT"');
+ expect(refusalPage).toContain('delivered=true');
for (const step of ['Record that this refusal was paged', 'Remember the refusal across ticks']) {
- const body = bugLane.slice(
- bugLane.indexOf(`- name: ${step}`),
- bugLane.indexOf(`- name: ${step}`) + 400,
- );
- expect(body, `${step} must gate on delivery`).toContain(
+ expect(bugLaneStep(step), `${step} must gate on delivery`).toContain(
"if: steps.page.outputs.delivered == 'true'",
);
}
});
+ test('an unchanged partial drop is paged once, not once per tick', () => {
+ // The drop page is a STANDING ANSWER on the same terms as the refusal
+ // above: the ref conflicts with the stable on every tick, so without a
+ // gate it re-sends an unchanged message every ~20 minutes. Observed
+ // 2026-08-21: four identical overnight pages, because the dispatch they
+ // accompanied could never land (anchor-drift) and so the candidate set
+ // never moved. verdict is `pass` on this path, which is exactly why the
+ // refusal's own gate does not cover it.
+ expect(bugLaneStep('Notify on a partial drop (armed only)')).toContain(
+ "steps.drop_paged_before.outputs.cache-hit != 'true'",
+ );
+ // Both cache steps, each bounded to itself. A restore key that stops
+ // tracking the signature is the per-tick flood this test is named for; if
+ // BOTH go constant it is permanent silence for every later drop.
+ for (const step of ['Has this drop already been paged?', 'Remember the drop across ticks']) {
+ expect(bugLaneStep(step), `${step} must key on the drop signature`).toContain(
+ 'key: bug-lane-drop-${{ steps.drop.outputs.sig }}',
+ );
+ }
+ });
+
+ test('the drop marker is gated on DELIVERY, not on the notify step succeeding', () => {
+ // Same contract as the refusal marker: the notify step cannot fail (an
+ // unset secret returns early, a dead webhook downgrades to a warning), so
+ // keying the marker on its conclusion would cache the signature even when
+ // nothing reached anyone and silence that drop permanently.
+ for (const step of ['Record that this drop was paged', 'Remember the drop across ticks']) {
+ expect(bugLaneStep(step), `${step} must gate on delivery`).toContain(
+ "if: steps.drop_page.outputs.delivered == 'true'",
+ );
+ }
+ });
+
+ test('the drop signature ignores the stable but tracks the dispatched subset', () => {
+ // Deliberate asymmetry, inherited from the refusal signature. The
+ // operator's move does not change when the stable rolls, and stables roll
+ // several times a day — including it would restore most of the flood for a
+ // fact already reported. A different surviving subset IS new news, so it
+ // stays in.
+ const sig = bugLaneStep('Drop signature');
+ expect(sig).toContain('"$DROPPED_REFS" "$SURVIVING_REFS"');
+ expect(sig).not.toContain('$STABLE');
+ });
+
+ test('a suppressed drop still leaves a trace in the run', () => {
+ // Same property the refusal path already pins: the suppressing tick has
+ // nothing else to show — no page, no marker write, green check — so an
+ // operator asking "is that drop still standing?" has only an invisible
+ // cache hit to infer it from. The gate is the COMPLEMENT of the notify
+ // step's; inverting it makes the note fire alongside the page and never on
+ // the tick it exists for.
+ const suppress = bugLaneStep('Note a suppressed drop');
+ expect(suppress).toContain("if: steps.drop_paged_before.outputs.cache-hit == 'true'");
+ expect(suppress).toContain('>> "$GITHUB_STEP_SUMMARY"');
+ });
+
+ test('the drop page states its delivery on every path', () => {
+ // The markers key on `delivered`, so the step has to write it whether or
+ // not a webhook exists. An early `exit 0` on the no-webhook branch leaves
+ // it unset — inert today, since unset and 'false' both fail the gate, but
+ // it makes the marker contract depend on a GHA default rather than on a
+ // value this step always states.
+ const notify = bugLaneStep('Notify on a partial drop (armed only)');
+ expect(notify).toContain('echo "delivered=${delivered}" >> "$GITHUB_OUTPUT"');
+ expect(notify).not.toContain('exit 0');
+ });
+
+ test('a disarmed lane cannot post the drop page', () => {
+ // The notify step used to carry `BUG_LANE_ARMED` in its own `if:`. Now that
+ // it gates on the signature instead, the armed check survives in exactly
+ // ONE place — and it is the only thing keeping a log-only lane from posting
+ // to Slack. Losing it there is silent: every other assertion here stays
+ // green while a disarmed lane starts paging.
+ expect(bugLaneStep('Drop signature')).toContain("env.BUG_LANE_ARMED == 'true'");
+ });
+
test('the one page it does send says the following silence is deliberate', () => {
// Paging once and then going quiet is indistinguishable from resolved
// unless the message says so.
diff --git a/.github/workflows/bug-lane.yml b/.github/workflows/bug-lane.yml
index ecbd778c2..08ca0ee71 100644
--- a/.github/workflows/bug-lane.yml
+++ b/.github/workflows/bug-lane.yml
@@ -314,11 +314,63 @@ jobs:
-f mode=cherry-pick -f fix_refs="$FIX_REFS" -f dry_run=false -f dispatched_by="$SELF_URL"
echo "::notice::bug-lane: dispatched point-release (cherry-pick) of ${FIX_REFS} over ${STABLE}."
+ # A partial drop is a STANDING ANSWER too, exactly like the refusal below:
+ # the same ref conflicts with the same stable on every tick, so the
+ # notification re-derives an unchanged fact every ~20 minutes. Observed
+ # 2026-08-21: four identical overnight pages while a point-release
+ # dispatch that could never land (anchor-drift) kept the candidate set
+ # frozen, so nothing about the message could change on its own.
+ #
+ # The stable is deliberately OUT of the signature, matching the refusal's
+ # reasoning one step down: the operator's move — "investigate whether any
+ # needs a manual point release" — does not change when the stable rolls.
+ # The surviving set IS in it, because a different dispatched subset is
+ # genuinely new news rather than the same fact restated.
+ - name: Drop signature
+ id: drop
+ if: env.BUG_LANE_ARMED == 'true' && steps.verify.outputs.verdict == 'pass' && steps.verify.outputs.dropped_refs != ''
+ env:
+ DROPPED_REFS: ${{ steps.verify.outputs.dropped_refs }}
+ SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}
+ run: |
+ set -euo pipefail
+ sig="$(printf '%s|%s' "$DROPPED_REFS" "$SURVIVING_REFS" | sha256sum | cut -c1-32)"
+ echo "sig=${sig}" >> "$GITHUB_OUTPUT"
+ echo "marker=${RUNNER_TEMP}/bug-lane-drop-marker" >> "$GITHUB_OUTPUT"
+
+ - name: Has this drop already been paged?
+ id: drop_paged_before
+ if: steps.drop.outputs.sig != ''
+ uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+ with:
+ path: ${{ steps.drop.outputs.marker }}
+ key: bug-lane-drop-${{ steps.drop.outputs.sig }}
+ lookup-only: true
+
+ # Same reason the refusal notes its suppression: a tick that goes quiet
+ # otherwise finishes green with no trace, leaving an operator asking "is
+ # that drop still standing?" with only an invisible cache hit to infer it
+ # from. The run summary costs no notification and answers it in place.
+ - name: Note a suppressed drop
+ if: steps.drop_paged_before.outputs.cache-hit == 'true'
+ env:
+ SIG: ${{ steps.drop.outputs.sig }}
+ DROPPED_REFS: ${{ steps.verify.outputs.dropped_refs }}
+ run: |
+ set -euo pipefail
+ {
+ echo "### Bug lane — drop still standing, page suppressed"
+ echo "Dropped \`${DROPPED_REFS}\`, signature \`${SIG}\`, already paged on an earlier tick."
+ echo "The channel stays quiet by design; this re-pages only if the dropped or dispatched set changes."
+ } >> "$GITHUB_STEP_SUMMARY"
+ echo "::notice::bug-lane: drop ${SIG} already paged; suppressing the duplicate page."
+
- name: Notify on a partial drop (armed only)
+ id: drop_page
# A drop with a surviving batch is verdict=pass, so the refusal page
# never fires — yet a bug fix silently deferred to the 24h path is the
# lane's most common non-trivial outcome and must not be log-only.
- if: env.BUG_LANE_ARMED == 'true' && steps.verify.outputs.verdict == 'pass' && steps.verify.outputs.dropped_refs != ''
+ if: steps.drop.outputs.sig != '' && steps.drop_paged_before.outputs.cache-hit != 'true'
env:
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
@@ -327,16 +379,43 @@ jobs:
STABLE: ${{ steps.evaluate.outputs.stable }}
run: |
set -uo pipefail
+ # `delivered` is the ONLY thing the markers may key on, so it is
+ # written on EVERY path — no early exit. An unset output and an
+ # explicit 'false' both fail the `== 'true'` gate today, so this is
+ # inert either way; it is written out because the contract is what
+ # the markers depend on, and an implicit GHA default is a thinner
+ # thing to depend on than a value this step always states. Same shape
+ # as the refusal step below, deliberately.
+ delivered=false
WEBHOOK_URL="${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}"
if [[ -z "$WEBHOOK_URL" ]]; then
echo "::notice::No Slack webhook secret set — drop notification skipped."
- exit 0
+ else
+ RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+ TEXT="⚠️ Bug lane dropped ${DROPPED_REFS} (cherry-pick conflict with ${STABLE}); they ride their cycle's 24h path — investigate whether any needs a manual point release. Dispatched the self-contained subset ${SURVIVING_REFS}. ${RUN_URL}"
+ payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"
+ if curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL"; then
+ delivered=true
+ else
+ echo "::warning::bug-lane drop notification failed to POST — the dispatch itself is unaffected, and the drop stays un-marked so the next tick pages it again."
+ fi
fi
- RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
- TEXT="⚠️ Bug lane dropped ${DROPPED_REFS} (cherry-pick conflict with ${STABLE}); they ride their cycle's 24h path — investigate whether any needs a manual point release. Dispatched the self-contained subset ${SURVIVING_REFS}. ${RUN_URL}"
- payload="$(jq -nc --arg t "$TEXT" '{text: $t}')"
- curl -sS --fail -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK_URL" \
- || echo "::warning::bug-lane drop notification failed to POST — the dispatch itself is unaffected."
+ echo "delivered=${delivered}" >> "$GITHUB_OUTPUT"
+
+ # Gated on DELIVERY, not on the step's conclusion — same contract as the
+ # refusal marker below. Writing it after a failed POST would turn a
+ # transient Slack outage into permanent silence for that drop, which is
+ # strictly worse than the per-tick flood this replaces.
+ - name: Record that this drop was paged
+ if: steps.drop_page.outputs.delivered == 'true'
+ run: printf '%s\n' "${{ steps.drop.outputs.sig }}" > "${{ steps.drop.outputs.marker }}"
+
+ - name: Remember the drop across ticks
+ if: steps.drop_page.outputs.delivered == 'true'
+ uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+ with:
+ path: ${{ steps.drop.outputs.marker }}
+ key: bug-lane-drop-${{ steps.drop.outputs.sig }}
# A refusal is a STANDING ANSWER, not an event. The refused ref stays in
# the pending pile and re-qualifies and re-conflicts on every tick, so a