Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Stop the OK fast-tier alarm flapping and re-smoking failed betas (#5242)
Andrew Mikofalvy4513b91c5217
diff --git a/.github/scripts/evaluate-smoke-alarm.mjs b/.github/scripts/evaluate-smoke-alarm.mjsindex c065dcf2c..992ff6b27 100644--- a/.github/scripts/evaluate-smoke-alarm.mjs+++ b/.github/scripts/evaluate-smoke-alarm.mjs@@ -9,6 +9,10 @@ export const CONSECUTIVE_NON_PASS_THRESHOLD = 3;export const STALE_FAST_TIER_WINDOW_DAYS = 14;const SMOKE_JOB_NAME = "Smoke the fast-tier candidate's DMG";+const EVALUATE_JOB_NAME = 'Evaluate 24h soak + business-hours gate';+const REMEMBERED_FAILURE_STEP_NAME =+ 'Skip the fast-tier candidate whose DMG already failed the smoke';+const SMOKE_OR_DISPATCH_STAGE = 'Smoke or dispatch';const DISPATCH_STEP_NAME = 'Dispatch promote-stable for the smoke-proven candidate';const DISPATCH_RECEIPT_STEP_NAME = 'Record a successful fast-tier dispatch';@@ -49,10 +53,19 @@ export function evaluateAlarm({return { alarm: reasons.length > 0, reasons };}+const REMEMBERED_SKIP = Symbol('remembered-skip');+export function buildHistory({ runs, jobsForRun }) {- return runs.map((run) => {+ const history = runs.map((run) => {const jobs = jobsForRun(run.databaseId ?? run.id) ?? [];const smoke = jobs.find((j) => j.name === SMOKE_JOB_NAME);+ const evaluate = jobs.find((j) => j.name === EVALUATE_JOB_NAME);+ const rememberedFailure = (evaluate?.steps ?? []).find(+ (s) => s.name === REMEMBERED_FAILURE_STEP_NAME,+ );+ if (rememberedFailure?.conclusion === 'success') {+ return REMEMBERED_SKIP;+ }if (!smoke ||(smoke.status && smoke.status !== 'completed') ||@@ -75,9 +88,34 @@ export function buildHistory({ runs, jobsForRun }) {failureStage: promoted? null: ((smoke.steps ?? []).find((s) => s.conclusion === 'failure')?.name ??- 'Smoke or dispatch'),+ SMOKE_OR_DISPATCH_STAGE),};});+ return resolveRememberedSkips(history, runs);+}++function resolveRememberedSkips(history, runs) {+ const resolved = [...history];+ let lastSmokedVerdict = null;+ for (let i = resolved.length - 1; i >= 0; i -= 1) {+ const entry = resolved[i];+ if (entry !== REMEMBERED_SKIP) {+ if (entry.qualified) lastSmokedVerdict = entry.verdict;+ continue;+ }+ const at = runs[i].createdAt;+ resolved[i] =+ lastSmokedVerdict === 'pass'+ ? { at, qualified: false, verdict: null, promoted: false }+ : {+ at,+ qualified: true,+ verdict: 'non-pass',+ promoted: false,+ failureStage: SMOKE_OR_DISPATCH_STAGE,+ };+ }+ return resolved;}export function alarmObservation({ history, nowMs, armed }) {@@ -100,6 +138,27 @@ export function classifyHistoryFailure(message) {const GH_CALL_TIMEOUT_MS = 30_000;+export const RUN_LIST_ARGS = [+ 'run',+ 'list',+ '--workflow=select-beta-to-promote.yml',+ '--limit',+ '60',+ '--json',+ 'databaseId,createdAt,status',+];++export function completedRunsNewestFirst(runs) {+ return runs+ .filter((r) => r.status === 'completed')+ .toSorted((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt));+}++export function listingIncludesRun(runs, runId) {+ if (!runId) return true;+ return runs.some((r) => String(r.databaseId ?? r.id) === String(runId));+}+function ghJson(args) {return JSON.parse(execFileSync('gh', args, { encoding: 'utf8', timeout: GH_CALL_TIMEOUT_MS }));}@@ -108,19 +167,15 @@ function main() {const repo = process.env.GITHUB_REPOSITORY || 'inkeep/open-knowledge';let history = [];try {- const runs = ghJson([- 'run',- 'list',- '--workflow=select-beta-to-promote.yml',- '--limit',- '60',- '--status',- 'completed',- '--json',- 'databaseId,createdAt',- ]);+ const listed = ghJson(RUN_LIST_ARGS);+ if (!listingIncludesRun(listed, process.env.GITHUB_RUN_ID)) {+ console.log(+ `::warning::The run listing does not include this run (${process.env.GITHUB_RUN_ID}), so it is not the current history; skipping the aggregate alarm this tick.`,+ );+ return;+ }history = buildHistory({- runs,+ runs: completedRunsNewestFirst(listed),jobsForRun: (id) => ghJson(['api', `repos/${repo}/actions/runs/${id}/jobs`]).jobs,});} catch (err) {diff --git a/.github/scripts/evaluate-smoke-alarm.test.mjs b/.github/scripts/evaluate-smoke-alarm.test.mjsindex bc4a457a2..7de98ef40 100644--- a/.github/scripts/evaluate-smoke-alarm.test.mjs+++ b/.github/scripts/evaluate-smoke-alarm.test.mjs@@ -7,7 +7,10 @@ import {buildHistory,CONSECUTIVE_NON_PASS_THRESHOLD,classifyHistoryFailure,+ completedRunsNewestFirst,evaluateAlarm,+ listingIncludesRun,+ RUN_LIST_ARGS,STALE_FAST_TIER_WINDOW_DAYS,} from './evaluate-smoke-alarm.mjs';@@ -241,6 +244,8 @@ describe('buildHistory', () => {);expect(wf).toContain("Smoke the fast-tier candidate's DMG");expect(wf).toContain('Dispatch promote-stable for the smoke-proven candidate');+ expect(wf).toContain('name: Evaluate 24h soak + business-hours gate');+ expect(wf).toContain('- name: Skip the fast-tier candidate whose DMG already failed the smoke');});test('the alarm pages Slack and never Discord', () => {@@ -381,3 +386,191 @@ describe('buildHistory run-id fallback', () => {).toBe(false);});});++describe('run history listing', () => {+ test('never asks GitHub to filter by status, which returns an arbitrary old page', () => {+ expect(RUN_LIST_ARGS).not.toContain('--status');+ expect(RUN_LIST_ARGS.join(' ')).toContain('status');+ });++ test('keeps only completed runs, newest first', () => {+ const runs = [+ { databaseId: 1, createdAt: daysAgo(3), status: 'completed' },+ { databaseId: 2, createdAt: daysAgo(0), status: 'in_progress' },+ { databaseId: 3, createdAt: daysAgo(1), status: 'completed' },+ { databaseId: 4, createdAt: daysAgo(2), status: 'queued' },+ ];+ expect(completedRunsNewestFirst(runs).map((r) => r.databaseId)).toEqual([3, 1]);+ });++ test('a listing that does not contain the evaluating run is not current history', () => {+ const runs = [+ { databaseId: 10, createdAt: daysAgo(20), status: 'completed' },+ { databaseId: 11, createdAt: daysAgo(20), status: 'completed' },+ ];+ expect(listingIncludesRun(runs, '99')).toBe(false);+ expect(listingIncludesRun([...runs, { databaseId: 99, status: 'in_progress' }], '99')).toBe(+ true,+ );+ });++ test('outside Actions there is no run id to anchor on, so the listing is accepted', () => {+ expect(listingIncludesRun([], undefined)).toBe(true);+ });+});++describe('a remembered smoke failure stays visible to the alarm', () => {+ const evaluateJob = (conclusion) => ({+ name: 'Evaluate 24h soak + business-hours gate',+ status: 'completed',+ conclusion: 'success',+ steps: [+ {+ name: 'Skip the fast-tier candidate whose DMG already failed the smoke',+ conclusion,+ },+ ],+ });+ const skippedSmoke = {+ name: "Smoke the fast-tier candidate's DMG",+ status: 'completed',+ conclusion: 'skipped',+ steps: [],+ };++ test('a tick that skipped a remembered failure is a qualified non-pass at the smoke stage', () => {+ const [entry] = buildHistory({+ runs: [{ databaseId: 1, createdAt: daysAgo(1) }],+ jobsForRun: () => [evaluateJob('success'), skippedSmoke],+ });+ expect(entry).toEqual({+ at: daysAgo(1),+ qualified: true,+ verdict: 'non-pass',+ promoted: false,+ failureStage: 'Smoke or dispatch',+ });+ });++ test('a tick with no remembered failure and no smoke did not qualify', () => {+ const [entry] = buildHistory({+ runs: [{ databaseId: 1, createdAt: daysAgo(1) }],+ jobsForRun: () => [evaluateJob('skipped'), skippedSmoke],+ });+ expect(entry.qualified).toBe(false);+ });++ test('remembered skips after a successful re-smoke do not reopen the incident', () => {+ const passedSmoke = {+ name: "Smoke the fast-tier candidate's DMG",+ status: 'completed',+ conclusion: 'success',+ steps: [+ { name: 'Dispatch promote-stable for the smoke-proven candidate', conclusion: 'success' },+ { name: 'Record a successful fast-tier dispatch', conclusion: 'success' },+ ],+ };+ const runs = Array.from({ length: 5 }, (_, i) => ({+ databaseId: i + 1,+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),+ }));+ const history = buildHistory({+ runs,+ jobsForRun: (id) =>+ id === 5 ? [evaluateJob('skipped'), passedSmoke] : [evaluateJob('success'), skippedSmoke],+ });+ expect(history.filter((h) => h.qualified)).toHaveLength(1);+ expect(alarmObservation({ history, nowMs: NOW, armed: true })).toMatchObject({+ alarm: false,+ incident: '',+ });+ });++ test('remembered skips after a failed smoke still count', () => {+ const failedSmoke = {+ name: "Smoke the fast-tier candidate's DMG",+ status: 'completed',+ conclusion: 'success',+ steps: [+ { name: 'Dispatch promote-stable for the smoke-proven candidate', conclusion: 'skipped' },+ { name: 'Record a successful fast-tier dispatch', conclusion: 'skipped' },+ ],+ };+ const runs = Array.from({ length: 4 }, (_, i) => ({+ databaseId: i + 1,+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),+ }));+ const history = buildHistory({+ runs,+ jobsForRun: (id) =>+ id === 4 ? [evaluateJob('skipped'), failedSmoke] : [evaluateJob('success'), skippedSmoke],+ });+ expect(history.filter((h) => h.qualified)).toHaveLength(4);+ expect(alarmObservation({ history, nowMs: NOW, armed: true }).alarm).toBe(true);+ });++ test.each([+ {+ name: 'the newest older smoke failed although an older one passed',+ ticks: ['skip', 'skip', 'skip', 'fail', 'pass'],+ alarm: true,+ },+ {+ name: 'a newer failed smoke of another beta does not decide older skips',+ ticks: ['fail', 'skip', 'skip', 'skip', 'pass'],+ alarm: false,+ },+ {+ name: 'an unqualified tick between the pass and the skips does not reset the rule',+ ticks: ['skip', 'skip', 'skip', 'none', 'pass'],+ alarm: false,+ },+ ])(+ 'a remembered skip is decided by the newest smoke older than it: $name',+ ({ ticks, alarm }) => {+ const smokeJob = (dispatched) => ({+ name: "Smoke the fast-tier candidate's DMG",+ status: 'completed',+ conclusion: 'success',+ steps: [+ {+ name: 'Dispatch promote-stable for the smoke-proven candidate',+ conclusion: dispatched ? 'success' : 'skipped',+ },+ {+ name: 'Record a successful fast-tier dispatch',+ conclusion: dispatched ? 'success' : 'skipped',+ },+ ],+ });+ const jobsFor = {+ skip: [evaluateJob('success'), skippedSmoke],+ none: [evaluateJob('skipped'), skippedSmoke],+ pass: [evaluateJob('skipped'), smokeJob(true)],+ fail: [evaluateJob('skipped'), smokeJob(false)],+ };+ const runs = ticks.map((_, i) => ({+ databaseId: i + 1,+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),+ }));+ const history = buildHistory({ runs, jobsForRun: (id) => jobsFor[ticks[id - 1]] });+ expect(alarmObservation({ history, nowMs: NOW, armed: true }).alarm).toBe(alarm);+ },+ );++ test('a broken beta keeps alarming after its only smoke attempt has left the sample', () => {+ const runs = Array.from({ length: 60 }, (_, i) => ({+ databaseId: i + 1,+ createdAt: new Date(NOW - i * 10 * 60 * 1000).toISOString(),+ }));+ const history = buildHistory({+ runs,+ jobsForRun: () => [evaluateJob('success'), skippedSmoke],+ });+ expect(alarmObservation({ history, nowMs: NOW, armed: true })).toMatchObject({+ alarm: true,+ observed: true,+ incident: 'Smoke or dispatch',+ });+ });+});diff --git a/.github/scripts/release-cascade-shape.test.mjs b/.github/scripts/release-cascade-shape.test.mjsindex 64de65b4f..8d5238cc4 100644--- a/.github/scripts/release-cascade-shape.test.mjs+++ b/.github/scripts/release-cascade-shape.test.mjs@@ -940,6 +940,70 @@ describe('every release-pipeline post prefers the releases webhook', () => {);});+ test('a beta whose DMG failed the smoke is remembered by tag and not re-smoked on later ticks', () => {+ const { jobs } = parse(selectBeta);+ const step = (job, name) => {+ const found = jobs[job].steps.find((s) => s.name === name);+ if (!found)+ throw new Error(`select-beta-to-promote.yml job ${job} has no step named ${name}`);+ return found;+ };+ expect(jobs.evaluate['runs-on']).toBe('ubuntu-latest');+ expect(jobs.evaluate.outputs.fast_tier_candidate).toBe(+ '${{ steps.nominate.outputs.fast_tier_candidate }}',+ );+ const lookup = step('evaluate', 'Look up an earlier smoke failure for the fast-tier candidate');+ expect(lookup.id).toBe('prior-failure');+ expect(lookup.if).toContain("github.event_name != 'workflow_dispatch'");+ expect(lookup.with).toEqual({+ path: 'fast-tier-smoke-failed',+ key: 'fast-tier-smoke-failed-v1-${{ steps.select.outputs.fast_tier_candidate }}',+ 'lookup-only': true,+ });+ expect(+ step('evaluate', 'Skip the fast-tier candidate whose DMG already failed the smoke').if,+ ).toBe("steps.prior-failure.outputs.cache-hit == 'true'");++ const nominate = step('evaluate', 'Nominate the fast-tier candidate for smoking');+ expect(nominate.env.ALREADY_FAILED).toBe('${{ steps.prior-failure.outputs.cache-hit }}');+ const nominated = (candidate, alreadyFailed) => {+ const dir = mkdtempSync(join(tmpdir(), 'ok-nominate-'));+ try {+ const out = join(dir, 'out');+ writeFileSync(out, '');+ execFileSync('bash', ['-c', nominate.run], {+ env: {+ ...process.env,+ CANDIDATE: candidate,+ ALREADY_FAILED: alreadyFailed,+ GITHUB_OUTPUT: out,+ },+ });+ return readFileSync(out, 'utf8');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ };+ expect(nominated('v1.2.3-beta.0', '')).toBe('fast_tier_candidate=v1.2.3-beta.0\n');+ expect(nominated('v1.2.3-beta.0', 'false')).toBe('fast_tier_candidate=v1.2.3-beta.0\n');+ expect(nominated('v1.2.3-beta.0', 'true')).toBe('fast_tier_candidate=\n');+ expect(nominated('', '')).toBe('fast_tier_candidate=\n');++ expect(jobs['smoke-fast-tier-candidate'].outputs.verdict).toBe(+ '${{ steps.smoke.outputs.verdict }}',+ );+ const remember = jobs['remember-smoke-failure'];+ expect(remember['runs-on']).toBe('ubuntu-latest');+ expect(remember.needs).toEqual(['evaluate', 'smoke-fast-tier-candidate']);+ expect(remember.if).toBe(+ "always() && needs.smoke-fast-tier-candidate.outputs.verdict == 'fail'",+ );+ expect(step('remember-smoke-failure', 'Save the failure marker').with).toEqual({+ path: 'fast-tier-smoke-failed',+ key: 'fast-tier-smoke-failed-v1-${{ needs.evaluate.outputs.fast_tier_candidate }}',+ });+ });+test('bug verification provisions the native runtime before testing the stable tree', () => {const setup = bugLaneVerify.indexOf('- name: Setup uv for ACP package acquisition tests');expect(setup).toBeGreaterThan(-1);diff --git a/.github/workflows/select-beta-to-promote.yml b/.github/workflows/select-beta-to-promote.ymlindex 10bfbbddf..5d002c717 100644--- a/.github/workflows/select-beta-to-promote.yml+++ b/.github/workflows/select-beta-to-promote.yml@@ -84,12 +84,12 @@ name: Select beta to promote# (3) the script stops at the first already-shipped beta (commit contained in the# latest stable) and never selects one whose changes are already released.#-# FAST TIER (FR5a), BUILT BUT UNARMED: a second job smokes an under-soaked-# beta's DMG and, only on a clean smoke, dispatches promote-stable early. It is-# gated on `fast_tier_candidate`, which the selection script writes empty today,-# so it never runs and adds zero per-tick cost. A fail or an error verdict only-# ever declines the early promotion — the 24h path is untouched and the gate can-# never block a release. Arming steps are documented in RELEASES.md.+# FAST TIER (FR5a), ARMED: a second job smokes an under-soaked beta's DMG and,+# only on a clean smoke, dispatches promote-stable early. It is gated on+# `fast_tier_candidate`, which is empty on ticks where nothing qualifies, and on+# ticks whose candidate's DMG already failed the smoke. A fail or an error+# verdict only ever declines the early promotion — the 24h path is untouched and+# the gate can never block a release. The operator record is RELEASES.md.on:schedule:@@ -168,7 +168,7 @@ jobs:# the macOS smoke leg keys off it. Empty on ticks where nothing qualifies,# which skips the smoke leg at zero cost — and empty on every tick when# FAST_TIER_ARMED is "false".- fast_tier_candidate: ${{ steps.select.outputs.fast_tier_candidate }}+ fast_tier_candidate: ${{ steps.nominate.outputs.fast_tier_candidate }}tier: ${{ steps.select.outputs.tier }}steps:- name: Business-hours gate (America/New_York)@@ -241,6 +241,34 @@ jobs:# Logic + tests: .github/scripts/select-beta-to-promote.{mjs,test.mjs}.run: node .github/scripts/select-beta-to-promote.mjs+ - name: Look up an earlier smoke failure for the fast-tier candidate+ id: prior-failure+ if: steps.select.outputs.fast_tier_candidate != '' && github.event_name != 'workflow_dispatch'+ uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4+ with:+ path: fast-tier-smoke-failed+ key: fast-tier-smoke-failed-v1-${{ steps.select.outputs.fast_tier_candidate }}+ lookup-only: true++ - name: Skip the fast-tier candidate whose DMG already failed the smoke+ if: steps.prior-failure.outputs.cache-hit == 'true'+ env:+ CANDIDATE: ${{ steps.select.outputs.fast_tier_candidate }}+ run: echo "::notice::${CANDIDATE}'s DMG already failed the fast-tier smoke, so it is not smoked again; it promotes on the 24h tier. A workflow_dispatch run re-smokes it."++ - name: Nominate the fast-tier candidate for smoking+ id: nominate+ env:+ CANDIDATE: ${{ steps.select.outputs.fast_tier_candidate }}+ ALREADY_FAILED: ${{ steps.prior-failure.outputs.cache-hit }}+ run: |+ set -euo pipefail+ if [[ "$ALREADY_FAILED" == "true" ]]; then+ echo "fast_tier_candidate=" >> "$GITHUB_OUTPUT"+ else+ echo "fast_tier_candidate=${CANDIDATE}" >> "$GITHUB_OUTPUT"+ fi+# Hours-gated, standard tier only. `target` is always the 24h selection —# a fast-tier cut never reaches this step; it is nominated as# `fast_tier_candidate` and promotes (around the clock) only through the@@ -323,6 +351,8 @@ jobs:# Apple Silicon: the DMG is a Universal build, but launching it needs macOS.runs-on: macos-26timeout-minutes: 30+ outputs:+ verdict: ${{ steps.smoke.outputs.verdict }}steps:- name: Checkoutuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -399,6 +429,27 @@ jobs:echo "::notice::Fast tier refused for ${CANDIDATE} (verdict=${VERDICT:-unknown}). The 24h tier is unaffected; this beta promotes normally once it has soaked."echo "Fast tier refused for ${CANDIDATE}; verdict=${VERDICT:-unknown}. The aggregate incident reporter owns Slack notifications." >> "$GITHUB_STEP_SUMMARY"+ remember-smoke-failure:+ name: Remember the candidate's failed smoke+ needs: [evaluate, smoke-fast-tier-candidate]+ if: always() && needs.smoke-fast-tier-candidate.outputs.verdict == 'fail'+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - name: Write the failure marker+ env:+ CANDIDATE: ${{ needs.evaluate.outputs.fast_tier_candidate }}+ run: |+ set -euo pipefail+ mkdir -p fast-tier-smoke-failed+ echo "$CANDIDATE" > fast-tier-smoke-failed/candidate++ - name: Save the failure marker+ uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4+ with:+ path: fast-tier-smoke-failed+ key: fast-tier-smoke-failed-v1-${{ needs.evaluate.outputs.fast_tier_candidate }}+# Aggregate silent-failure alarm.## The per-cut alert in desktop-release.yml covers a gate that BITES. This@@ -406,12 +457,6 @@ jobs:# always-refusing, or a fast tier that has gone inert. Both look exactly like# a calm week.#- # Silent by default and by design. With the fast tier unarmed the smoke job- # never runs, the derived history is empty, and the evaluator reports no- # alarm — an intentionally disarmed tier is not a broken one, and an alarm- # that fired continuously while the tier was off would train responders to- # ignore it.- #aggregate-smoke-alarm:name: Aggregate DMG-smoke alarmif: github.event_name == 'schedule'