Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(core): drain capped CPU alerts durably without repeats [A6,H16]

Corey Olson8471461af688
diff --git a/src/board.js b/src/board.js
index 38e71024..2a652453 100644
--- a/src/board.js
+++ b/src/board.js
@@ -243,6 +243,7 @@ import {
} from './rigor-notices.js';
import { RIGOR_NOTICE_ENGINE } from './rigor-source.js';
import { RIGOR_AUTO_ENGINE } from './rigor-auto-engine.js';
+import { CPU_SENTINEL_ENGINE } from './cpu-sentinel-engine.js';
import {
deriveRigorCause,
recordAutomaticRigorDrafts,
@@ -1702,8 +1703,9 @@ export function shout(board, message) {
/** Record one current health snapshot atomically, notifying each new or resolved episode once [A6]. */
export function recordHealthFindings(board, { agentId, observedRevision, findings }) {
return atomic(board, () => {
- if ((board.executionEngineVersion ?? ENGINE_VERSION) < HEALTH_SENTINEL_ENGINE) {
- throw new Refused('ENGINE_VERSION', `sentinel receipts require engine ${HEALTH_SENTINEL_ENGINE}; upgrade Pullboard before scanning`);
+ const minimum = (Array.isArray(findings) && findings.some(finding => finding?.cause === 'AGENT_CPU')) ? CPU_SENTINEL_ENGINE : HEALTH_SENTINEL_ENGINE;
+ if ((board.executionEngineVersion ?? ENGINE_VERSION) < minimum) {
+ throw new Refused('ENGINE_VERSION', `sentinel receipts require engine ${minimum}; upgrade Pullboard before scanning`);
}
if (!board.db.prepare('SELECT 1 FROM agent WHERE agent_id = ?').get(agentId)) {
throw new Refused('NOT_JOINED', 'join this board before recording a health scan: pullboard join <lane>');
@@ -1719,7 +1721,8 @@ export function recordHealthFindings(board, { agentId, observedRevision, finding
const after = new Map(active.map(finding => [healthFindingIdentity(finding), finding]));
const added = active.filter(finding => !before.has(healthFindingIdentity(finding)));
const resolved = previous.filter(finding => !after.has(healthFindingIdentity(finding)));
- if (!added.length && !resolved.length) return { added: [], resolved: [] };
+ const pending = last ? JSON.parse(last.event_detail).pendingCpu ?? [] : [];
+ if (!added.length && !resolved.length && !pending.length) return { added: [], resolved: [] };
const agents = listAgents(board);
const lanes = [...new Set(agents.map(agent => agent.agent_lane))];
const engine = board.executionEngineVersion ?? ENGINE_VERSION;
@@ -1731,9 +1734,9 @@ export function recordHealthFindings(board, { agentId, observedRevision, finding
owner: finding.owner, commit: item?.item_commit ?? null, engine: HEALTH_SENTINEL_ENGINE,
});
};
- deliverHealthNotices({ resolved, added, engine, log, coordinator: COORDINATOR, shout: (to, text) => insertShout(board, { from: 'sentinel', to, text, lanes }),
+ const deferred = deliverHealthNotices({ resolved, added, pending, engine, log, coordinator: COORDINATOR, shout: (to, text) => insertShout(board, { from: 'sentinel', to, text, lanes }),
targets: finding => healthTargets(finding, { engine, coordinator: COORDINATOR, person: PERSON, lanes, agents }) });
- logEvent(board, agentId, 'health-scan', null, { active, engine: HEALTH_SENTINEL_ENGINE });
+ logEvent(board, agentId, 'health-scan', null, { active, ...(deferred === undefined ? {} : { pendingCpu: deferred }), engine: HEALTH_SENTINEL_ENGINE });
return { added: added.map(finding => finding.key), resolved: resolved.map(finding => finding.key) };
});
}
diff --git a/src/cpu-sentinel-engine.js b/src/cpu-sentinel-engine.js
new file mode 100644
index 00000000..4ae0a93e
--- /dev/null
+++ b/src/cpu-sentinel-engine.js
@@ -0,0 +1,3 @@
+/** Durable capped CPU notification delivery; earlier health receipts keep their replay [A6,H16]. */
+import { unassignedEngine } from './engine-next.js';
+export const CPU_SENTINEL_ENGINE = unassignedEngine();
diff --git a/src/health-receipts.js b/src/health-receipts.js
index 839eaceb..8592006f 100644
--- a/src/health-receipts.js
+++ b/src/health-receipts.js
@@ -1,5 +1,6 @@
/** Durable work-health observations, separate from the person's decisions [A6,H16]. */
import { Refused } from './refused.js';
+import { CPU_SENTINEL_ENGINE } from './cpu-sentinel-engine.js';
/** The allocated engine for ordered sentinel observations; earlier moves keep their meaning. */
export const HEALTH_SENTINEL_ENGINE = 42;
@@ -66,17 +67,24 @@ export function healthTargets(finding, { engine, coordinator, person, lanes, age
* @param {{ resolved: any[], added: any[], engine: number, coordinator: string, targets: (finding: any) => string[],
* shout: (to: string, text: string) => void, log: (finding: any, cleared: boolean) => void }} run
*/
-export function deliverHealthNotices({ resolved, added, engine, coordinator, targets, shout, log }) {
+export function deliverHealthNotices({ resolved, added, pending = [], engine, coordinator, targets, shout, log }) {
const capped = engine >= HEALTH_CAP_ENGINE;
+ const durableCpu = engine >= CPU_SENTINEL_ENGINE;
+ const deferred = [];
let sent = 0;
let held = 0;
- for (const [finding, cleared] of [...resolved.map((entry) => [entry, true]), ...added.map((entry) => [entry, false])]) {
+ for (const [finding, cleared, retry] of [...(durableCpu ? pending : []).map(entry => [entry, false, true]),
+ ...resolved.map(entry => [entry, true, false]), ...added.map(entry => [entry, false, false])]) {
+ let coordinatorSent = false;
for (const to of targets(finding)) {
if (capped && sent >= HEALTH_SHOUT_CAP) { held += 1; continue; }
shout(to, healthFindingNotice(finding, cleared));
sent += 1;
+ if (to === coordinator) coordinatorSent = true;
}
- log(finding, cleared);
+ if (durableCpu && !cleared && finding.cause === 'AGENT_CPU' && !coordinatorSent) deferred.push(finding);
+ if (!retry) log(finding, cleared);
}
if (held) shout(coordinator, `Health: ${held} more notices from this run were held back, so one run never floods a feed; pullboard health lists every finding.`);
+ return durableCpu ? deferred : undefined;
}
diff --git a/src/health-sentinel.js b/src/health-sentinel.js
index b6f7980a..06422ee4 100644
--- a/src/health-sentinel.js
+++ b/src/health-sentinel.js
@@ -24,7 +24,7 @@ export function scanHealthSentinel(file, { root, config, now, cpu = agentCpuFind
const before = (last ? JSON.parse(last.event_detail).active : []).map(healthFindingIdentity).sort();
const after = normalizedHealthFindings(report.stuck).map(healthFindingIdentity).sort();
db.exec('COMMIT');
- if (JSON.stringify(before) === JSON.stringify(after)) return null;
+ if (JSON.stringify(before) === JSON.stringify(after) && !(last && JSON.parse(last.event_detail).pendingCpu?.length)) return null;
return { observedRevision, findings: report.stuck };
} finally { db.close(); }
}
diff --git a/src/moves/recordHealthFindings.js b/src/moves/recordHealthFindings.js
index bfda6c51..b86d2514 100644
--- a/src/moves/recordHealthFindings.js
+++ b/src/moves/recordHealthFindings.js
@@ -1,3 +1,4 @@
+import { CPU_SENTINEL_ENGINE } from '../cpu-sentinel-engine.js';
import { RIGOR_AUTO_ENGINE } from '../rigor-auto-engine.js';
/** Routing metadata owned by recordHealthFindings, discovered without a shared registry edit [C7,V4,H16]. */
import { HEALTH_SENTINEL_ENGINE } from '../health-receipts.js';
@@ -9,6 +10,7 @@ function minimumEngine(args, board, at) {
if (typeof args[1]?.cause === 'string' && args[1].cause.startsWith('conflict:')) return RIGOR_AUTO_ENGINE;
if (args[0]?.landingFence !== undefined) return LANDING_LEASE_ENGINE;
if (typeof args[1]?.cause === 'string' && args[1].cause.startsWith('sentinel:')) return BOARD_SENTINEL_ENGINE;
+ if (Array.isArray(args[0]?.findings) && args[0].findings.some(finding => finding?.cause === 'AGENT_CPU')) return CPU_SENTINEL_ENGINE;
return HEALTH_SENTINEL_ENGINE;
}
diff --git a/src/sentinel-cpu.js b/src/sentinel-cpu.js
index 7ed80871..1f0eaa43 100644
--- a/src/sentinel-cpu.js
+++ b/src/sentinel-cpu.js
@@ -69,8 +69,10 @@ export function agentCpuFindings(db, { platform = process.platform, run = native
if (!agent) return [];
const key = `${prefix}${row.pid}:${row.since}`;
// Durable prior alerts survive temporary observation gaps and observer restarts; PID birth prevents reuse collisions.
- const active = previous.some(finding => finding.key === key);
+ const recorded = previous.find(finding => finding.key === key);
+ const active = Boolean(recorded);
if (!active && db.prepare("SELECT 1 FROM event WHERE event_kind='health-failure' AND json_extract(event_detail,'$.key')=? LIMIT 1").get(key)) return [];
+ if (recorded) return [{ ...recorded, owner: { who: recorded.owner } }];
return [{ key, cause: 'AGENT_CPU', item: null, subject: `pid ${row.pid}`, title: row.name, since: row.since,
why: `${row.cpu.toFixed(2)} CPU minutes used on ${host} in agent worktree ${agent.path}`, next: 'inspect the owning agent process before deciding whether to stop it', owner: { who: agent.agent_id } }];
})];
diff --git a/test/sentinel-cpu.test.js b/test/sentinel-cpu.test.js
index 9ae04311..c7d882de 100644
--- a/test/sentinel-cpu.test.js
+++ b/test/sentinel-cpu.test.js
@@ -7,6 +7,8 @@ import { test } from 'node:test';
import * as store from '../src/board.js';
import { runFixtureGit } from './fixture-child.js';
import { scanHealthSentinel } from '../src/health-sentinel.js';
+import { CPU_SENTINEL_ENGINE } from '../src/cpu-sentinel-engine.js';
+import { HEALTH_SHOUT_CAP } from '../src/health-receipts.js';
// CPU time is observed, not burned: the native utility port supplies an exact ps/lsof transcript.
test('the sentinel names an agent process past 20 CPU minutes once [A6,H16]', async t => {
@@ -15,18 +17,19 @@ test('the sentinel names an agent process past 20 CPU minutes once [A6,H16]', as
const root = join(directory, 'repo'), work = join(directory, 'core');
mkdirSync(root); mkdirSync(work);
runFixtureGit(['init', '-q', '-b', 'main'], { cwd: root });
- const file = join(directory, 'board.sqlite'), board = store.openBoard(file);
+ const file = join(directory, 'board.sqlite');
+ let board = store.openBoard(file);
store.register(board, { lane: 'coordinator', path: root });
const seat = store.register(board, { lane: 'core', path: work });
t.after(() => { store.closeBoard(board); rmSync(directory, { recursive: true }); });
- let birth = 'Mon Oct 19 12:00:00 2026', cpu = '21:30.00', cwd = work, present = true, unavailable = false, host = 'host-a';
+ let birth = 'Mon Oct 19 12:00:00 2026', cpu = '21:30.00', cwd = work, present = true, unavailable = false, host = 'host-a', pids = [4321];
const calls = [];
const run = (command, args) => {
calls.push([command, args]);
if (unavailable) throw new Error('native observation unavailable');
- if (command === '/bin/ps') return present ? ` 4321 ${cpu} ${birth} /usr/bin/git\n 4322 19:59.99 ${birth} /usr/bin/node\n` : '';
+ if (command === '/bin/ps') return present ? pids.map(pid => ` ${pid} ${cpu} ${birth} /usr/bin/git\n`).join('') + ` 9999 19:59.99 ${birth} /usr/bin/node\n` : '';
assert.equal(command, '/usr/sbin/lsof');
- return `p4321\nn${cwd}\n`;
+ return pids.map(pid => `p${pid}\nn${cwd}\n`).join('');
};
const observe = () => agentCpuFindings(board.db, { platform: 'darwin', run, host });
const scan = () => {
@@ -38,6 +41,9 @@ test('the sentinel names an agent process past 20 CPU minutes once [A6,H16]', as
assert.equal(notices().length, 1);
for (const value of ['4321', 'git', seat, '21.50 CPU minutes']) assert.ok(notices()[0].shout_text.includes(value), value);
assert.equal(notices()[0].shout_from, 'sentinel');
+ const moved = join(directory, 'another-seat'); mkdirSync(moved);
+ store.register(board, { lane: 'core', path: moved }); cwd = moved; scan(); cwd = work;
+ assert.equal(notices().length, 1, 'the same process changing worktree does not create another alert');
unavailable = true; scan(); unavailable = false;
assert.equal(store.allShouts(board).filter(row => row.shout_text.startsWith('Resolved:') && row.shout_text.includes('AGENT_CPU')).length, 0, 'an unavailable observation does not claim resolution');
cpu = '01:22:30'; scan(); scan();
@@ -61,6 +67,26 @@ test('the sentinel names an agent process past 20 CPU minutes once [A6,H16]', as
host = 'host-a'; scan(); host = 'host-b'; scan();
assert.equal(notices().length, beforeHost + 1, 'alternating hosts do not repeat each other’s alert');
assert.equal(store.allShouts(board).filter(row => row.shout_text.startsWith('Resolved:') && row.shout_text.includes('AGENT_CPU')).length, resolved, 'one host cannot resolve another host’s observation');
+ host = 'cap-host'; birth = 'Mon Oct 19 15:00:00 2026'; pids = Array.from({ length: 12 }, (_, index) => 5001 + index);
+ const beforeCap = notices().length;
+ const failureCount = () => store.events(board).filter(event => event.event_kind === 'health-failure').length;
+ const failuresBefore = failureCount();
+ const cappedScan = () => {
+ const count = store.allShouts(board).length; scan();
+ assert.ok(store.allShouts(board).length - count <= HEALTH_SHOUT_CAP + 1, 'cap ten plus one summary remains unchanged');
+ };
+ cappedScan();
+ assert.equal(notices().length - beforeCap, 5, 'sixth and later coordinator alerts wait rather than exceed the cap');
+ store.closeBoard(board); board = store.openBoard(file);
+ cappedScan(); cappedScan();
+ for (const pid of pids) assert.equal(notices().filter(row => row.shout_text.includes(`pid ${pid} `)).length, 1, `PID ${pid} is named exactly once after restart`);
+ assert.equal(notices().length - beforeCap, pids.length);
+ assert.equal(failureCount() - failuresBefore, pids.length, 'retrying capped delivery does not invent recurring health failures');
+ const eventsAfter = store.events(board).length; scan();
+ assert.equal(store.events(board).length, eventsAfter, 'a drained delivery queue makes no more events');
+ board.executionEngineVersion = CPU_SENTINEL_ENGINE - 1;
+ assert.throws(() => store.recordHealthFindings(board, { agentId: 'coordinator', observedRevision: eventsAfter, findings: observe() }), { code: 'ENGINE_VERSION' });
+ delete board.executionEngineVersion;
assert.ok(calls.some(([command, args]) => command === '/bin/ps' && args.includes('time=')));
assert.ok(calls.some(([command]) => command === '/usr/sbin/lsof'));
});