an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
fix(worker-wake): hooks are activity evidence too, and the stall rule is off for an unobservable agent
Gastón Péchieuf3d2e324b541
diff --git a/src/main/index.ts b/src/main/index.tsindex 66dc140c..2035035c 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -5165,8 +5165,12 @@ function runWorkerWakeBeat(): void {paused: snap.paused,halted: snap.halted,// A turn the CLI demonstrably took: a tool span, or a usage sample WITH- // tokens. The zero-token sample stamped at session start is not one.+ // tokens. The zero-token sample stamped at session start is not one —+ // but it does prove the CLI exports telemetry (Claude Code only), which+ // is what lets the stall rule read "no turn" as evidence. Other engines+ // show their turns through hook events, which the watchdog hears itself.lastActivityAt: activityEvidenceAt({ usage: telemetry.getAgentUsage(agentId), spans: telemetry.getSpans(agentId) }),+ hasTelemetry: telemetry.getAgentUsage(agentId) !== null,oldestMailAt});}@@ -5192,7 +5196,8 @@ function runWorkerWakeBeat(): void {if (!workerWake.shouldReportHold(f.agentId, now)) continue;const hold = workerWake.explain(f, now);const quiet = f.lastOutputAt > 0 ? `${Math.round((now - f.lastOutputAt) / 1000)}s` : 'never';- const active = f.lastActivityAt && f.lastActivityAt > 0 ? `${Math.round((now - f.lastActivityAt) / 1000)}s ago` : 'never';+ const activeAt = Math.max(f.lastActivityAt ?? 0, workerWake.turnHookAt(f.agentId));+ const active = activeAt > 0 ? `${Math.round((now - activeAt) / 1000)}s ago` : 'never';console.warn(`[worker-wake] holding ${f.agentId}: ${hold} (mail pending ${Math.round(mailAge / 1000)}s, pty quiet ${quiet}, last activity ${active})`);}}diff --git a/src/main/workerWake.ts b/src/main/workerWake.tsindex 166ac3ee..7faae481 100644--- a/src/main/workerWake.ts+++ b/src/main/workerWake.ts@@ -54,7 +54,16 @@ export const WORKER_WAKE_HITL_REARM_MS = 5 * 60_000;* still subject to paused/halted/HITL/boot-grace/cooldown. Observed live* 2026-09-06: a worker sat 17 minutes on its work order with 0 tokens and no* transcript until the human typed "read your inbox" by hand; this watchdog- * never fired. */+ * never fired.+ *+ * "Session activity" is a tool span or a usage sample with tokens (telemetry,+ * which only Claude Code exports) OR a hook event that proves a turn+ * (UserPromptSubmit / PreToolUse / PostToolUse / Stop — every engine the+ * harness shims sends those). The rule is OFF for an agent that has produced+ * neither a telemetry sample nor a single hook event: with no channel that+ * could ever show a turn, "no activity" is not evidence of anything, and a+ * Codex/Gemini/grok worker would otherwise read as stalled forever and be+ * nudged every cooldown while working — the repeated nudging #368 removed. */export const WORKER_WAKE_STALL_MS = 90_000;/** Minimum age of pending mail before a held worker is reported in the log. */export const WORKER_WAKE_REPORT_MS = 60_000;@@ -81,6 +90,27 @@ export function classifyHook(event: string | undefined, message: string | undefireturn null;}+/** A hook event that proves the CLI took a turn — the activity signal every+ * engine the harness shims produces (Codex, Gemini, grok, … are mapped onto+ * these names in hive.ts), unlike telemetry, which only Claude Code exports.+ * SessionStart is the CLI coming up, not a turn: a worker whose boot nudge was+ * lost has exactly that and nothing else. Notification is the CLI waiting. */+export function isTurnHook(event: string | undefined): boolean {+ switch (event) {+ case 'UserPromptSubmit':+ case 'PreToolUse':+ case 'PostToolUse':+ case 'PostToolUseFailure':+ case 'Stop':+ case 'StopFailure':+ case 'SubagentStart':+ case 'SubagentStop':+ return true;+ default:+ return false;+ }+}+/** One worker's live facts, gathered by the caller each beat. */export interface WorkerWakeFacts {/** Worker agent id (god is never a candidate). */@@ -100,6 +130,11 @@ export interface WorkerWakeFacts {/** Timestamp of the agent's last telemetry usage sample — the CLI's own* evidence of a turn — or 0/undefined when it has never reported one. */lastActivityAt?: number;+ /** True when the telemetry collector holds ANY usage sample for the agent+ * (even the zero-token one stamped at session start): its CLI exports+ * telemetry, so a missing turn there means something. Only Claude Code+ * does; for every other engine the hooks are the activity channel. */+ hasTelemetry?: boolean;/** created_at of the OLDEST undrained inbox message, or 0/undefined when* unknown (the stall rule then stays off — fail closed, as before). */oldestMailAt?: number;@@ -160,23 +195,50 @@ export class WorkerWakeWatchdog {private announcedInboxIds = new Map<string, Set<string>>();/** agentId → timestamp of the last needsHuman hook notification. */private lastHumanNeedsAt = new Map<string, number>();+ /** agentId → timestamp of its last hook event of ANY kind: the agent's hooks+ * are alive, so a missing turn hook means something. */+ private hookSeenAt = new Map<string, number>();+ /** agentId → timestamp of its last hook event that proves a turn. */+ private lastTurnHookAt = new Map<string, number>();/** Record a PTY spawn so its boot sequence is left alone. */noteSpawn(ptyId: string, at = Date.now()): void {this.spawnedAt.set(ptyId, at);}- /** Feed hook events (from HookServer) so a HITL prompt blocks nudges. */+ /** Feed hook events (from HookServer): a HITL prompt blocks nudges, and any+ * turn-proving event is activity the stall rule credits — the one channel+ * every engine has, telemetry being Claude-only. */noteHook(agentId: string | undefined, event: string | undefined, message: string | undefined, at = Date.now()): void {if (!agentId) return;+ this.hookSeenAt.set(agentId, at);+ if (isTurnHook(event) && at > (this.lastTurnHookAt.get(agentId) ?? 0)) this.lastTurnHookAt.set(agentId, at);if (classifyHook(event, message) === 'needsHuman') this.lastHumanNeedsAt.set(agentId, at);}+ /** When the agent's hooks last proved a turn, or 0 when they never have. */+ turnHookAt(agentId: string): number {+ return this.lastTurnHookAt.get(agentId) ?? 0;+ }++ /** The stall rule on everything known: the beat's telemetry evidence plus+ * the hooks' — and OFF for an agent that has produced neither a telemetry+ * sample nor a single hook event, because such an agent cannot show a turn+ * even when it takes one (see WORKER_WAKE_STALL_MS). */+ private isStalled(f: WorkerWakeFacts, now: number): boolean {+ const observable = !!f.hasTelemetry || this.hookSeenAt.has(f.agentId);+ if (!observable) return false;+ const lastActivityAt = Math.max(f.lastActivityAt ?? 0, this.turnHookAt(f.agentId));+ return isStalledWorker({ ...f, lastActivityAt }, now);+ }+/** Forget per-agent state (e.g. the agent's PTY was closed). */forget(agentId: string, ptyId?: string): void {this.lastNudgeAt.delete(agentId);this.announcedInboxIds.delete(agentId);this.lastHumanNeedsAt.delete(agentId);+ this.hookSeenAt.delete(agentId);+ this.lastTurnHookAt.delete(agentId);this.lastHoldReportAt.delete(agentId);if (ptyId) this.spawnedAt.delete(ptyId);}@@ -196,7 +258,7 @@ export class WorkerWakeWatchdog {if (f.autoDeliveryPaused) return 'delivery-paused';if (f.paused) return 'paused';if (f.halted) return 'halted';- const stalled = isStalledWorker(f, now);+ const stalled = this.isStalled(f, now);if (f.lastOutputAt <= 0 && !stalled) return 'booting'; // never produced output → still bootingif (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS && !stalled) return 'mid-turn';const spawned = this.spawnedAt.get(f.ptyId) ?? 0;diff --git a/test/worker-wake-stall.test.cjs b/test/worker-wake-stall.test.cjsindex 9e8e652c..ef27b270 100644--- a/test/worker-wake-stall.test.cjs+++ b/test/worker-wake-stall.test.cjs@@ -18,6 +18,7 @@ const {WorkerWakeWatchdog,isStalledWorker,activityEvidenceAt,+ isTurnHook,WORKER_WAKE_IDLE_MS,WORKER_WAKE_STALL_MS,WORKER_WAKE_COOLDOWN_MS,@@ -26,8 +27,9 @@ const {const NOW = 10_000_000; // far enough from epoch that "20 minutes ago" stays positive-/** A worker whose terminal is chatty (output 1s ago) — the shape that used to- * read as "mid-turn" forever. */+/** A Claude worker whose terminal is chatty (output 1s ago) — the shape that+ * used to read as "mid-turn" forever. Claude Code exports telemetry, so the+ * collector holds a sample for it (the zero-token boot one at least). */function chatty(overrides = {}) {return {agentId: 'stanley',@@ -37,10 +39,18 @@ function chatty(overrides = {}) {autoDeliveryPaused: false,paused: false,halted: false,+ hasTelemetry: true,...overrides};}+/** The same worker on an engine that exports no telemetry (Codex, Gemini,+ * grok, …): the collector never sees a sample, so `lastActivityAt` is always+ * 0 and the hooks are the only activity channel. */+function codex(overrides = {}) {+ return chatty({ agentId: 'kevin', ptyId: 'pty-kevin', hasTelemetry: false, lastActivityAt: 0, ...overrides });+}+function watchdog() {const w = new WorkerWakeWatchdog();w.noteSpawn('pty-stanley', NOW - 20 * 60_000); // spawned 20 minutes ago@@ -118,6 +128,78 @@ test('mail younger than the stall window keeps the original quiet-output rule',assert.equal(w.explain(quiet, NOW), null);});+test('isTurnHook: prompt submits, tool boundaries and stops prove a turn; boot, idle and compaction do not', () => {+ for (const e of ['UserPromptSubmit', 'PreToolUse', 'PostToolUse', 'PostToolUseFailure', 'Stop', 'StopFailure', 'SubagentStart', 'SubagentStop']) {+ assert.equal(isTurnHook(e), true, e);+ }+ for (const e of ['SessionStart', 'SessionEnd', 'Notification', 'PreCompact', 'PostCompact', 'PermissionDenied', 'Unknown', undefined]) {+ assert.equal(isTurnHook(e), false, String(e));+ }+});++// Activity evidence used to come from telemetry alone, and only Claude Code+// exports it. For a Codex / Gemini / grok worker it was always 0, so once its+// mail was 90 s old the stall rule fired on every tick and the worker was nudged+// every cooldown WHILE WORKING, until the file moved to .done — the repeated+// nudging #368 removed. Now the hooks (which every shimmed engine sends) are+// evidence too, and an agent with neither channel is never called stalled.+test('an engine with no telemetry and no hook event is never stalled: the pre-stall rules apply unchanged (fail closed, #368)', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);+ const working = codex({ oldestMailAt: NOW - 10 * 60_000 });+ assert.equal(w.explain(working, NOW), 'mid-turn', 'its chatty terminal is all we have, and it says busy');+ assert.deepEqual(w.decide([working], NOW), []);+ const quiet = codex({ oldestMailAt: NOW - 10 * 60_000, lastOutputAt: NOW - WORKER_WAKE_IDLE_MS - 1 });+ assert.deepEqual(w.decide([quiet], NOW), ['kevin'], 'a quiet terminal is nudged once, as before');+ assert.equal(w.explain(quiet, NOW + WORKER_WAKE_COOLDOWN_MS + 1), 'announced', 'and never again for the same mail — no stall override without evidence');+});++test('a hook-only engine whose hooks show no turn since the mail landed is stalled and nudged every cooldown', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);+ const mailAt = NOW - 10 * 60_000;+ // Its hooks are alive (the CLI came up after the mail) but never proved a turn.+ w.noteHook('kevin', 'SessionStart', undefined, mailAt + 1_000);+ w.noteHook('kevin', 'Notification', 'Codex is waiting for your input', mailAt + 2_000);+ const f = codex({ oldestMailAt: mailAt });+ assert.equal(w.explain(f, NOW), null, 'SessionStart and an idle Notification are not turns');+ assert.deepEqual(w.decide([f], NOW), ['kevin']);+ assert.deepEqual(w.decide([f], NOW + WORKER_WAKE_COOLDOWN_MS), ['kevin'], 'retried every cooldown until it acts');+ // A turn hook BEFORE the mail does not count either.+ const w2 = new WorkerWakeWatchdog();+ w2.noteSpawn('pty-kevin', NOW - 20 * 60_000);+ w2.noteHook('kevin', 'PostToolUse', undefined, mailAt - 1);+ assert.equal(w2.explain(f, NOW), null);+});++test('a turn hook after the mail is activity: the worker is working it, not stalled (no repeated nudging)', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);+ const mailAt = NOW - 10 * 60_000;+ w.noteHook('kevin', 'PreToolUse', undefined, NOW - 5_000);+ const f = codex({ oldestMailAt: mailAt });+ assert.equal(w.explain(f, NOW), 'mid-turn');+ assert.deepEqual(w.decide([f], NOW), []);+ assert.equal(w.turnHookAt('kevin'), NOW - 5_000, 'the beat can log when the hooks last proved a turn');+ // Hook evidence also covers a Claude worker whose telemetry is dead.+ const c = new WorkerWakeWatchdog();+ c.noteSpawn('pty-stanley', NOW - 20 * 60_000);+ c.noteHook('stanley', 'UserPromptSubmit', undefined, NOW - 3_000);+ assert.equal(c.explain(chatty({ oldestMailAt: mailAt, lastActivityAt: 0 }), NOW), 'mid-turn');+});++test('forget() drops the hook memory: a re-spawned agent starts unobservable again', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);+ w.noteHook('kevin', 'SessionStart', undefined, NOW - 9 * 60_000);+ const f = codex({ oldestMailAt: NOW - 10 * 60_000 });+ assert.equal(w.explain(f, NOW), null, 'stalled while its hooks are known');+ w.forget('kevin', 'pty-kevin');+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);+ assert.equal(w.explain(f, NOW), 'mid-turn', 'no hook seen this session → the stall rule is off');+ assert.equal(w.turnHookAt('kevin'), 0);+});+test('facts without the new fields behave exactly as before (fail closed)', () => {const w = watchdog();assert.equal(w.explain(chatty(), NOW), 'mid-turn');