Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Stop the OK fast-tier alarm flapping and re-smoking failed betas (#5242)

Andrew Mikofalvy4513b91c5217
diff --git a/.github/scripts/evaluate-smoke-alarm.mjs b/.github/scripts/evaluate-smoke-alarm.mjs
index c065dcf2c..992ff6b27 100644
--- a/.github/scripts/evaluate-smoke-alarm.mjs
+++ b/.github/scripts/evaluate-smoke-alarm.mjs
@@ -9,6 +9,10 @@ export const CONSECUTIVE_NON_PASS_THRESHOLD = 3;
export const STALE_FAST_TIER_WINDOW_DAYS = 14;
const SMOKE_JOB_NAME = "Smoke the fast-tier candidate's DMG";
+const EVALUATE_JOB_NAME = 'Evaluate 24h soak + business-hours gate';
+const REMEMBERED_FAILURE_STEP_NAME =
+ 'Skip the fast-tier candidate whose DMG already failed the smoke';
+const SMOKE_OR_DISPATCH_STAGE = 'Smoke or dispatch';
const DISPATCH_STEP_NAME = 'Dispatch promote-stable for the smoke-proven candidate';
const DISPATCH_RECEIPT_STEP_NAME = 'Record a successful fast-tier dispatch';
@@ -49,10 +53,19 @@ export function evaluateAlarm({
return { alarm: reasons.length > 0, reasons };
}
+const REMEMBERED_SKIP = Symbol('remembered-skip');
+
export function buildHistory({ runs, jobsForRun }) {
- return runs.map((run) => {
+ const history = runs.map((run) => {
const jobs = jobsForRun(run.databaseId ?? run.id) ?? [];
const smoke = jobs.find((j) => j.name === SMOKE_JOB_NAME);
+ const evaluate = jobs.find((j) => j.name === EVALUATE_JOB_NAME);
+ const rememberedFailure = (evaluate?.steps ?? []).find(
+ (s) => s.name === REMEMBERED_FAILURE_STEP_NAME,
+ );
+ if (rememberedFailure?.conclusion === 'success') {
+ return REMEMBERED_SKIP;
+ }
if (
!smoke ||
(smoke.status && smoke.status !== 'completed') ||
@@ -75,9 +88,34 @@ export function buildHistory({ runs, jobsForRun }) {
failureStage: promoted
? null
: ((smoke.steps ?? []).find((s) => s.conclusion === 'failure')?.name ??
- 'Smoke or dispatch'),
+ SMOKE_OR_DISPATCH_STAGE),
};
});
+ return resolveRememberedSkips(history, runs);
+}
+
+function resolveRememberedSkips(history, runs) {
+ const resolved = [...history];
+ let lastSmokedVerdict = null;
+ for (let i = resolved.length - 1; i >= 0; i -= 1) {
+ const entry = resolved[i];
+ if (entry !== REMEMBERED_SKIP) {
+ if (entry.qualified) lastSmokedVerdict = entry.verdict;
+ continue;
+ }
+ const at = runs[i].createdAt;
+ resolved[i] =
+ lastSmokedVerdict === 'pass'
+ ? { at, qualified: false, verdict: null, promoted: false }
+ : {
+ at,
+ qualified: true,
+ verdict: 'non-pass',
+ promoted: false,
+ failureStage: SMOKE_OR_DISPATCH_STAGE,
+ };
+ }
+ return resolved;
}
export function alarmObservation({ history, nowMs, armed }) {
@@ -100,6 +138,27 @@ export function classifyHistoryFailure(message) {
const GH_CALL_TIMEOUT_MS = 30_000;
+export const RUN_LIST_ARGS = [
+ 'run',
+ 'list',
+ '--workflow=select-beta-to-promote.yml',
+ '--limit',
+ '60',
+ '--json',
+ 'databaseId,createdAt,status',
+];
+
+export function completedRunsNewestFirst(runs) {
+ return runs
+ .filter((r) => r.status === 'completed')
+ .toSorted((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt));
+}
+
+export function listingIncludesRun(runs, runId) {
+ if (!runId) return true;
+ return runs.some((r) => String(r.databaseId ?? r.id) === String(runId));
+}
+
function ghJson(args) {
return JSON.parse(execFileSync('gh', args, { encoding: 'utf8', timeout: GH_CALL_TIMEOUT_MS }));
}
@@ -108,19 +167,15 @@ function main() {
const repo = process.env.GITHUB_REPOSITORY || 'inkeep/open-knowledge';
let history = [];
try {
- const runs = ghJson([
- 'run',
- 'list',
- '--workflow=select-beta-to-promote.yml',
- '--limit',
- '60',
- '--status',
- 'completed',
- '--json',
- 'databaseId,createdAt',
- ]);
+ const listed = ghJson(RUN_LIST_ARGS);
+ if (!listingIncludesRun(listed, process.env.GITHUB_RUN_ID)) {
+ console.log(
+ `::warning::The run listing does not include this run (${process.env.GITHUB_RUN_ID}), so it is not the current history; skipping the aggregate alarm this tick.`,
+ );
+ return;
+ }
history = buildHistory({
- runs,
+ runs: completedRunsNewestFirst(listed),
jobsForRun: (id) => ghJson(['api', `repos/${repo}/actions/runs/${id}/jobs`]).jobs,
});
} catch (err) {
diff --git a/.github/scripts/evaluate-smoke-alarm.test.mjs b/.github/scripts/evaluate-smoke-alarm.test.mjs
index bc4a457a2..7de98ef40 100644
--- a/.github/scripts/evaluate-smoke-alarm.test.mjs
+++ b/.github/scripts/evaluate-smoke-alarm.test.mjs
@@ -7,7 +7,10 @@ import {
buildHistory,
CONSECUTIVE_NON_PASS_THRESHOLD,
classifyHistoryFailure,
+ completedRunsNewestFirst,
evaluateAlarm,
+ listingIncludesRun,
+ RUN_LIST_ARGS,
STALE_FAST_TIER_WINDOW_DAYS,
} from './evaluate-smoke-alarm.mjs';
@@ -241,6 +244,8 @@ describe('buildHistory', () => {
);
expect(wf).toContain("Smoke the fast-tier candidate's DMG");
expect(wf).toContain('Dispatch promote-stable for the smoke-proven candidate');
+ expect(wf).toContain('name: Evaluate 24h soak + business-hours gate');
+ expect(wf).toContain('- name: Skip the fast-tier candidate whose DMG already failed the smoke');
});
test('the alarm pages Slack and never Discord', () => {
@@ -381,3 +386,191 @@ describe('buildHistory run-id fallback', () => {
).toBe(false);
});
});
+
+describe('run history listing', () => {
+ test('never asks GitHub to filter by status, which returns an arbitrary old page', () => {
+ expect(RUN_LIST_ARGS).not.toContain('--status');
+ expect(RUN_LIST_ARGS.join(' ')).toContain('status');
+ });
+
+ test('keeps only completed runs, newest first', () => {
+ const runs = [
+ { databaseId: 1, createdAt: daysAgo(3), status: 'completed' },
+ { databaseId: 2, createdAt: daysAgo(0), status: 'in_progress' },
+ { databaseId: 3, createdAt: daysAgo(1), status: 'completed' },
+ { databaseId: 4, createdAt: daysAgo(2), status: 'queued' },
+ ];
+ expect(completedRunsNewestFirst(runs).map((r) => r.databaseId)).toEqual([3, 1]);
+ });
+
+ test('a listing that does not contain the evaluating run is not current history', () => {
+ const runs = [
+ { databaseId: 10, createdAt: daysAgo(20), status: 'completed' },
+ { databaseId: 11, createdAt: daysAgo(20), status: 'completed' },
+ ];
+ expect(listingIncludesRun(runs, '99')).toBe(false);
+ expect(listingIncludesRun([...runs, { databaseId: 99, status: 'in_progress' }], '99')).toBe(
+ true,
+ );
+ });
+
+ test('outside Actions there is no run id to anchor on, so the listing is accepted', () => {
+ expect(listingIncludesRun([], undefined)).toBe(true);
+ });
+});
+
+describe('a remembered smoke failure stays visible to the alarm', () => {
+ const evaluateJob = (conclusion) => ({
+ name: 'Evaluate 24h soak + business-hours gate',
+ status: 'completed',
+ conclusion: 'success',
+ steps: [
+ {
+ name: 'Skip the fast-tier candidate whose DMG already failed the smoke',
+ conclusion,
+ },
+ ],
+ });
+ const skippedSmoke = {
+ name: "Smoke the fast-tier candidate's DMG",
+ status: 'completed',
+ conclusion: 'skipped',
+ steps: [],
+ };
+
+ test('a tick that skipped a remembered failure is a qualified non-pass at the smoke stage', () => {
+ const [entry] = buildHistory({
+ runs: [{ databaseId: 1, createdAt: daysAgo(1) }],
+ jobsForRun: () => [evaluateJob('success'), skippedSmoke],
+ });
+ expect(entry).toEqual({
+ at: daysAgo(1),
+ qualified: true,
+ verdict: 'non-pass',
+ promoted: false,
+ failureStage: 'Smoke or dispatch',
+ });
+ });
+
+ test('a tick with no remembered failure and no smoke did not qualify', () => {
+ const [entry] = buildHistory({
+ runs: [{ databaseId: 1, createdAt: daysAgo(1) }],
+ jobsForRun: () => [evaluateJob('skipped'), skippedSmoke],
+ });
+ expect(entry.qualified).toBe(false);
+ });
+
+ test('remembered skips after a successful re-smoke do not reopen the incident', () => {
+ const passedSmoke = {
+ name: "Smoke the fast-tier candidate's DMG",
+ status: 'completed',
+ conclusion: 'success',
+ steps: [
+ { name: 'Dispatch promote-stable for the smoke-proven candidate', conclusion: 'success' },
+ { name: 'Record a successful fast-tier dispatch', conclusion: 'success' },
+ ],
+ };
+ const runs = Array.from({ length: 5 }, (_, i) => ({
+ databaseId: i + 1,
+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),
+ }));
+ const history = buildHistory({
+ runs,
+ jobsForRun: (id) =>
+ id === 5 ? [evaluateJob('skipped'), passedSmoke] : [evaluateJob('success'), skippedSmoke],
+ });
+ expect(history.filter((h) => h.qualified)).toHaveLength(1);
+ expect(alarmObservation({ history, nowMs: NOW, armed: true })).toMatchObject({
+ alarm: false,
+ incident: '',
+ });
+ });
+
+ test('remembered skips after a failed smoke still count', () => {
+ const failedSmoke = {
+ name: "Smoke the fast-tier candidate's DMG",
+ status: 'completed',
+ conclusion: 'success',
+ steps: [
+ { name: 'Dispatch promote-stable for the smoke-proven candidate', conclusion: 'skipped' },
+ { name: 'Record a successful fast-tier dispatch', conclusion: 'skipped' },
+ ],
+ };
+ const runs = Array.from({ length: 4 }, (_, i) => ({
+ databaseId: i + 1,
+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),
+ }));
+ const history = buildHistory({
+ runs,
+ jobsForRun: (id) =>
+ id === 4 ? [evaluateJob('skipped'), failedSmoke] : [evaluateJob('success'), skippedSmoke],
+ });
+ expect(history.filter((h) => h.qualified)).toHaveLength(4);
+ expect(alarmObservation({ history, nowMs: NOW, armed: true }).alarm).toBe(true);
+ });
+
+ test.each([
+ {
+ name: 'the newest older smoke failed although an older one passed',
+ ticks: ['skip', 'skip', 'skip', 'fail', 'pass'],
+ alarm: true,
+ },
+ {
+ name: 'a newer failed smoke of another beta does not decide older skips',
+ ticks: ['fail', 'skip', 'skip', 'skip', 'pass'],
+ alarm: false,
+ },
+ {
+ name: 'an unqualified tick between the pass and the skips does not reset the rule',
+ ticks: ['skip', 'skip', 'skip', 'none', 'pass'],
+ alarm: false,
+ },
+ ])(
+ 'a remembered skip is decided by the newest smoke older than it: $name',
+ ({ ticks, alarm }) => {
+ const smokeJob = (dispatched) => ({
+ name: "Smoke the fast-tier candidate's DMG",
+ status: 'completed',
+ conclusion: 'success',
+ steps: [
+ {
+ name: 'Dispatch promote-stable for the smoke-proven candidate',
+ conclusion: dispatched ? 'success' : 'skipped',
+ },
+ {
+ name: 'Record a successful fast-tier dispatch',
+ conclusion: dispatched ? 'success' : 'skipped',
+ },
+ ],
+ });
+ const jobsFor = {
+ skip: [evaluateJob('success'), skippedSmoke],
+ none: [evaluateJob('skipped'), skippedSmoke],
+ pass: [evaluateJob('skipped'), smokeJob(true)],
+ fail: [evaluateJob('skipped'), smokeJob(false)],
+ };
+ const runs = ticks.map((_, i) => ({
+ databaseId: i + 1,
+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),
+ }));
+ const history = buildHistory({ runs, jobsForRun: (id) => jobsFor[ticks[id - 1]] });
+ expect(alarmObservation({ history, nowMs: NOW, armed: true }).alarm).toBe(alarm);
+ },
+ );
+
+ test('a broken beta keeps alarming after its only smoke attempt has left the sample', () => {
+ const runs = Array.from({ length: 60 }, (_, i) => ({
+ databaseId: i + 1,
+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),
+ }));
+ const history = buildHistory({
+ runs,
+ jobsForRun: () => [evaluateJob('success'), skippedSmoke],
+ });
+ expect(alarmObservation({ history, nowMs: NOW, armed: true })).toMatchObject({
+ alarm: true,
+ observed: true,
+ incident: 'Smoke or dispatch',
+ });
+ });
+});
diff --git a/.github/scripts/release-cascade-shape.test.mjs b/.github/scripts/release-cascade-shape.test.mjs
index 64de65b4f..8d5238cc4 100644
--- a/.github/scripts/release-cascade-shape.test.mjs
+++ b/.github/scripts/release-cascade-shape.test.mjs
@@ -940,6 +940,70 @@ describe('every release-pipeline post prefers the releases webhook', () => {
);
});
+ test('a beta whose DMG failed the smoke is remembered by tag and not re-smoked on later ticks', () => {
+ const { jobs } = parse(selectBeta);
+ const step = (job, name) => {
+ const found = jobs[job].steps.find((s) => s.name === name);
+ if (!found)
+ throw new Error(`select-beta-to-promote.yml job ${job} has no step named ${name}`);
+ return found;
+ };
+ expect(jobs.evaluate['runs-on']).toBe('ubuntu-latest');
+ expect(jobs.evaluate.outputs.fast_tier_candidate).toBe(
+ '${{ steps.nominate.outputs.fast_tier_candidate }}',
+ );
+ const lookup = step('evaluate', 'Look up an earlier smoke failure for the fast-tier candidate');
+ expect(lookup.id).toBe('prior-failure');
+ expect(lookup.if).toContain("github.event_name != 'workflow_dispatch'");
+ expect(lookup.with).toEqual({
+ path: 'fast-tier-smoke-failed',
+ key: 'fast-tier-smoke-failed-v1-${{ steps.select.outputs.fast_tier_candidate }}',
+ 'lookup-only': true,
+ });
+ expect(
+ step('evaluate', 'Skip the fast-tier candidate whose DMG already failed the smoke').if,
+ ).toBe("steps.prior-failure.outputs.cache-hit == 'true'");
+
+ const nominate = step('evaluate', 'Nominate the fast-tier candidate for smoking');
+ expect(nominate.env.ALREADY_FAILED).toBe('${{ steps.prior-failure.outputs.cache-hit }}');
+ const nominated = (candidate, alreadyFailed) => {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-nominate-'));
+ try {
+ const out = join(dir, 'out');
+ writeFileSync(out, '');
+ execFileSync('bash', ['-c', nominate.run], {
+ env: {
+ ...process.env,
+ CANDIDATE: candidate,
+ ALREADY_FAILED: alreadyFailed,
+ GITHUB_OUTPUT: out,
+ },
+ });
+ return readFileSync(out, 'utf8');
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ };
+ expect(nominated('v1.2.3-beta.0', '')).toBe('fast_tier_candidate=v1.2.3-beta.0\n');
+ expect(nominated('v1.2.3-beta.0', 'false')).toBe('fast_tier_candidate=v1.2.3-beta.0\n');
+ expect(nominated('v1.2.3-beta.0', 'true')).toBe('fast_tier_candidate=\n');
+ expect(nominated('', '')).toBe('fast_tier_candidate=\n');
+
+ expect(jobs['smoke-fast-tier-candidate'].outputs.verdict).toBe(
+ '${{ steps.smoke.outputs.verdict }}',
+ );
+ const remember = jobs['remember-smoke-failure'];
+ expect(remember['runs-on']).toBe('ubuntu-latest');
+ expect(remember.needs).toEqual(['evaluate', 'smoke-fast-tier-candidate']);
+ expect(remember.if).toBe(
+ "always() && needs.smoke-fast-tier-candidate.outputs.verdict == 'fail'",
+ );
+ expect(step('remember-smoke-failure', 'Save the failure marker').with).toEqual({
+ path: 'fast-tier-smoke-failed',
+ key: 'fast-tier-smoke-failed-v1-${{ needs.evaluate.outputs.fast_tier_candidate }}',
+ });
+ });
+
test('bug verification provisions the native runtime before testing the stable tree', () => {
const setup = bugLaneVerify.indexOf('- name: Setup uv for ACP package acquisition tests');
expect(setup).toBeGreaterThan(-1);
diff --git a/.github/workflows/select-beta-to-promote.yml b/.github/workflows/select-beta-to-promote.yml
index 10bfbbddf..5d002c717 100644
--- a/.github/workflows/select-beta-to-promote.yml
+++ b/.github/workflows/select-beta-to-promote.yml
@@ -84,12 +84,12 @@ name: Select beta to promote
# (3) the script stops at the first already-shipped beta (commit contained in the
# latest stable) and never selects one whose changes are already released.
#
-# FAST TIER (FR5a), BUILT BUT UNARMED: a second job smokes an under-soaked
-# beta's DMG and, only on a clean smoke, dispatches promote-stable early. It is
-# gated on `fast_tier_candidate`, which the selection script writes empty today,
-# so it never runs and adds zero per-tick cost. A fail or an error verdict only
-# ever declines the early promotion — the 24h path is untouched and the gate can
-# never block a release. Arming steps are documented in RELEASES.md.
+# FAST TIER (FR5a), ARMED: a second job smokes an under-soaked beta's DMG and,
+# only on a clean smoke, dispatches promote-stable early. It is gated on
+# `fast_tier_candidate`, which is empty on ticks where nothing qualifies, and on
+# ticks whose candidate's DMG already failed the smoke. A fail or an error
+# verdict only ever declines the early promotion — the 24h path is untouched and
+# the gate can never block a release. The operator record is RELEASES.md.
on:
schedule:
@@ -168,7 +168,7 @@ jobs:
# the macOS smoke leg keys off it. Empty on ticks where nothing qualifies,
# which skips the smoke leg at zero cost — and empty on every tick when
# FAST_TIER_ARMED is "false".
- fast_tier_candidate: ${{ steps.select.outputs.fast_tier_candidate }}
+ fast_tier_candidate: ${{ steps.nominate.outputs.fast_tier_candidate }}
tier: ${{ steps.select.outputs.tier }}
steps:
- name: Business-hours gate (America/New_York)
@@ -241,6 +241,34 @@ jobs:
# Logic + tests: .github/scripts/select-beta-to-promote.{mjs,test.mjs}.
run: node .github/scripts/select-beta-to-promote.mjs
+ - name: Look up an earlier smoke failure for the fast-tier candidate
+ id: prior-failure
+ if: steps.select.outputs.fast_tier_candidate != '' && github.event_name != 'workflow_dispatch'
+ uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+ with:
+ path: fast-tier-smoke-failed
+ key: fast-tier-smoke-failed-v1-${{ steps.select.outputs.fast_tier_candidate }}
+ lookup-only: true
+
+ - name: Skip the fast-tier candidate whose DMG already failed the smoke
+ if: steps.prior-failure.outputs.cache-hit == 'true'
+ env:
+ CANDIDATE: ${{ steps.select.outputs.fast_tier_candidate }}
+ run: echo "::notice::${CANDIDATE}'s DMG already failed the fast-tier smoke, so it is not smoked again; it promotes on the 24h tier. A workflow_dispatch run re-smokes it."
+
+ - name: Nominate the fast-tier candidate for smoking
+ id: nominate
+ env:
+ CANDIDATE: ${{ steps.select.outputs.fast_tier_candidate }}
+ ALREADY_FAILED: ${{ steps.prior-failure.outputs.cache-hit }}
+ run: |
+ set -euo pipefail
+ if [[ "$ALREADY_FAILED" == "true" ]]; then
+ echo "fast_tier_candidate=" >> "$GITHUB_OUTPUT"
+ else
+ echo "fast_tier_candidate=${CANDIDATE}" >> "$GITHUB_OUTPUT"
+ fi
+
# Hours-gated, standard tier only. `target` is always the 24h selection —
# a fast-tier cut never reaches this step; it is nominated as
# `fast_tier_candidate` and promotes (around the clock) only through the
@@ -323,6 +351,8 @@ jobs:
# Apple Silicon: the DMG is a Universal build, but launching it needs macOS.
runs-on: macos-26
timeout-minutes: 30
+ outputs:
+ verdict: ${{ steps.smoke.outputs.verdict }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -399,6 +429,27 @@ jobs:
echo "::notice::Fast tier refused for ${CANDIDATE} (verdict=${VERDICT:-unknown}). The 24h tier is unaffected; this beta promotes normally once it has soaked."
echo "Fast tier refused for ${CANDIDATE}; verdict=${VERDICT:-unknown}. The aggregate incident reporter owns Slack notifications." >> "$GITHUB_STEP_SUMMARY"
+ remember-smoke-failure:
+ name: Remember the candidate's failed smoke
+ needs: [evaluate, smoke-fast-tier-candidate]
+ if: always() && needs.smoke-fast-tier-candidate.outputs.verdict == 'fail'
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - name: Write the failure marker
+ env:
+ CANDIDATE: ${{ needs.evaluate.outputs.fast_tier_candidate }}
+ run: |
+ set -euo pipefail
+ mkdir -p fast-tier-smoke-failed
+ echo "$CANDIDATE" > fast-tier-smoke-failed/candidate
+
+ - name: Save the failure marker
+ uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+ with:
+ path: fast-tier-smoke-failed
+ key: fast-tier-smoke-failed-v1-${{ needs.evaluate.outputs.fast_tier_candidate }}
+
# Aggregate silent-failure alarm.
#
# The per-cut alert in desktop-release.yml covers a gate that BITES. This
@@ -406,12 +457,6 @@ jobs:
# always-refusing, or a fast tier that has gone inert. Both look exactly like
# a calm week.
#
- # Silent by default and by design. With the fast tier unarmed the smoke job
- # never runs, the derived history is empty, and the evaluator reports no
- # alarm — an intentionally disarmed tier is not a broken one, and an alarm
- # that fired continuously while the tier was off would train responders to
- # ignore it.
- #
aggregate-smoke-alarm:
name: Aggregate DMG-smoke alarm
if: github.event_name == 'schedule'