Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ci): stop the fast-tier smoke alarm paging about a tier that is switched off (#3048)

Andrew Mikofalvy65f5c458b112
diff --git a/.github/scripts/evaluate-smoke-alarm.mjs b/.github/scripts/evaluate-smoke-alarm.mjs
index d4ffeb9b0..c0d025059 100644
--- a/.github/scripts/evaluate-smoke-alarm.mjs
+++ b/.github/scripts/evaluate-smoke-alarm.mjs
@@ -43,10 +43,19 @@ const DISPATCH_STEP_NAME = 'Dispatch promote-stable for the smoke-proven candida
export function evaluateAlarm({
history,
nowMs,
+ armed = true,
consecutiveThreshold = CONSECUTIVE_NON_PASS_THRESHOLD,
windowDays = STALE_FAST_TIER_WINDOW_DAYS,
}) {
const reasons = [];
+ // Both conditions below are claims about an ARMED tier — "the gate looks
+ // persistently broken" and "the tier is armed and reaching nothing". Neither
+ // is a finding while the switch is off: with `FAST_TIER_ARMED=false` the
+ // selector forces `soak_tier=standard`, so `promoted` can never be true no
+ // matter how healthy the gate is, and condition 2 becomes structurally
+ // guaranteed the moment any cut qualifies. That fired hourly into a public
+ // channel about a tier nobody had turned on.
+ if (!armed) return { alarm: false, reasons };
const qualified = history.filter((h) => h.qualified);
// Condition 1 — leading run of non-pass verdicts among qualified cuts.
@@ -73,8 +82,10 @@ export function evaluateAlarm({
`${qualifiedInWindow.length} cut(s) qualified for the fast tier in the last ${windowDays} days but none was promoted through it — the tier is armed and reaching nothing`,
);
}
- // The disarmed case needs no branch: with nothing qualified, condition 1's
- // streak is 0 and condition 2's guard is false, so both stay silent.
+ // The disarmed case is handled by the `armed` short-circuit above, not by
+ // "nothing qualifies". `qualified` is derived from whether the fast-tier
+ // smoke job RAN, and that job is gated on a candidate existing rather than
+ // on the arming switch — so a disarmed tier still produces qualified cuts.
return { alarm: reasons.length > 0, reasons };
}
@@ -163,7 +174,11 @@ function main() {
);
}
- const { alarm, reasons } = evaluateAlarm({ history, nowMs: Date.now() });
+ // Default to DISARMED when the variable is absent. The workflow always
+ // passes it, so an absent value means an unexpected caller — and staying
+ // quiet is the safe failure for a channel-paging alarm.
+ const armed = process.env.FAST_TIER_ARMED === 'true';
+ const { alarm, reasons } = evaluateAlarm({ history, nowMs: Date.now(), armed });
if (!alarm) {
console.log('No aggregate smoke alarm: the fast tier is either healthy or intentionally off.');
} else {
diff --git a/.github/scripts/evaluate-smoke-alarm.test.mjs b/.github/scripts/evaluate-smoke-alarm.test.mjs
index a735dd9bd..fb2879a95 100644
--- a/.github/scripts/evaluate-smoke-alarm.test.mjs
+++ b/.github/scripts/evaluate-smoke-alarm.test.mjs
@@ -109,6 +109,30 @@ describe('a disarmed fast tier is not a broken one', () => {
const r = run(Array.from({ length: 40 }, (_, i) => notQualified({ at: daysAgo(i * 0.3) })));
expect(r.alarm).toBe(false);
});
+
+ test('qualified-but-never-promoted stays silent when the switch is off', () => {
+ // The shape that actually paged. `qualified` tracks whether the fast-tier
+ // smoke job RAN, and that job is gated on a candidate existing rather than
+ // on the arming switch — so a disarmed tier still produces qualified cuts,
+ // while `promoted` can never become true because the selector forces
+ // soak_tier=standard. Condition 2 is therefore structurally guaranteed the
+ // moment any cut qualifies, and it fired hourly into a public channel.
+ const history = [cut({ at: daysAgo(1) }), cut({ at: daysAgo(4) })];
+ expect(evaluateAlarm({ history, nowMs: NOW, armed: false })).toEqual({
+ alarm: false,
+ reasons: [],
+ });
+ // Same history, tier armed: this IS a finding and must still fire.
+ expect(evaluateAlarm({ history, nowMs: NOW, armed: true }).alarm).toBe(true);
+ });
+
+ test('a persistently broken gate is also not a finding while disarmed', () => {
+ const history = Array.from({ length: CONSECUTIVE_NON_PASS_THRESHOLD + 1 }, (_, i) =>
+ cut({ at: daysAgo(i + 1) }),
+ );
+ expect(evaluateAlarm({ history, nowMs: NOW, armed: false }).alarm).toBe(false);
+ expect(evaluateAlarm({ history, nowMs: NOW, armed: true }).alarm).toBe(true);
+ });
});
describe('both conditions can fire together', () => {
diff --git a/.github/workflows/select-beta-to-promote.yml b/.github/workflows/select-beta-to-promote.yml
index 7186e63b7..cf20fd01d 100644
--- a/.github/workflows/select-beta-to-promote.yml
+++ b/.github/workflows/select-beta-to-promote.yml
@@ -131,6 +131,17 @@ concurrency:
group: ok-auto-promote-stable
cancel-in-progress: true
+env:
+ # THE fast-tier arming switch, read by both the selector and the aggregate
+ # alarm. Hoisted to one place because the two must agree: the alarm's whole
+ # premise is "the tier is armed and reaching nothing", which is only a real
+ # finding when the tier is in fact armed. Two literals would drift and the
+ # alarm would page about a tier that is switched off.
+ # Flipping this to "true" requires a reviewed PR — read the "Soak tiers"
+ # section of RELEASES.md first: arming has prerequisites, and it is an owner
+ # decision.
+ FAST_TIER_ARMED: "false"
+
jobs:
evaluate:
name: Evaluate 24h soak + business-hours gate
@@ -202,7 +213,7 @@ jobs:
# it did before the tier existed. Flipping it to "true" requires a
# reviewed PR — read the "Soak tiers" section of RELEASES.md first:
# arming has prerequisites, and it is an owner decision.
- FAST_TIER_ARMED: "false"
+ FAST_TIER_ARMED: ${{ env.FAST_TIER_ARMED }}
# Optional. Absent (the current state) the bug-link lookup reports
# "unresolvable" and the tier degrades to standard, which is the
# expected steady state until an operator provisions the secret.
@@ -402,6 +413,11 @@ jobs:
id: alarm
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Both alarm conditions are statements about an ARMED tier. Passing
+ # the switch in lets the evaluator stay silent while it is off,
+ # instead of inferring "armed" from history that a disarmed tier
+ # still produces.
+ FAST_TIER_ARMED: ${{ env.FAST_TIER_ARMED }}
run: node .github/scripts/evaluate-smoke-alarm.mjs
# Routed to the same channels as the per-cut smoke alert, not to a