Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

fix(worker-wake): hooks are activity evidence too, and the stall rule is off for an unobservable agent

Gastón Péchieuf3d2e324b541
diff --git a/src/main/index.ts b/src/main/index.ts
index 66dc140c..2035035c 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -5165,8 +5165,12 @@ function runWorkerWakeBeat(): void {
paused: snap.paused,
halted: snap.halted,
// A turn the CLI demonstrably took: a tool span, or a usage sample WITH
- // tokens. The zero-token sample stamped at session start is not one.
+ // tokens. The zero-token sample stamped at session start is not one —
+ // but it does prove the CLI exports telemetry (Claude Code only), which
+ // is what lets the stall rule read "no turn" as evidence. Other engines
+ // show their turns through hook events, which the watchdog hears itself.
lastActivityAt: activityEvidenceAt({ usage: telemetry.getAgentUsage(agentId), spans: telemetry.getSpans(agentId) }),
+ hasTelemetry: telemetry.getAgentUsage(agentId) !== null,
oldestMailAt
});
}
@@ -5192,7 +5196,8 @@ function runWorkerWakeBeat(): void {
if (!workerWake.shouldReportHold(f.agentId, now)) continue;
const hold = workerWake.explain(f, now);
const quiet = f.lastOutputAt > 0 ? `${Math.round((now - f.lastOutputAt) / 1000)}s` : 'never';
- const active = f.lastActivityAt && f.lastActivityAt > 0 ? `${Math.round((now - f.lastActivityAt) / 1000)}s ago` : 'never';
+ const activeAt = Math.max(f.lastActivityAt ?? 0, workerWake.turnHookAt(f.agentId));
+ const active = activeAt > 0 ? `${Math.round((now - activeAt) / 1000)}s ago` : 'never';
console.warn(`[worker-wake] holding ${f.agentId}: ${hold} (mail pending ${Math.round(mailAge / 1000)}s, pty quiet ${quiet}, last activity ${active})`);
}
}
diff --git a/src/main/workerWake.ts b/src/main/workerWake.ts
index 166ac3ee..7faae481 100644
--- a/src/main/workerWake.ts
+++ b/src/main/workerWake.ts
@@ -54,7 +54,16 @@ export const WORKER_WAKE_HITL_REARM_MS = 5 * 60_000;
* still subject to paused/halted/HITL/boot-grace/cooldown. Observed live
* 2026-09-06: a worker sat 17 minutes on its work order with 0 tokens and no
* transcript until the human typed "read your inbox" by hand; this watchdog
- * never fired. */
+ * never fired.
+ *
+ * "Session activity" is a tool span or a usage sample with tokens (telemetry,
+ * which only Claude Code exports) OR a hook event that proves a turn
+ * (UserPromptSubmit / PreToolUse / PostToolUse / Stop — every engine the
+ * harness shims sends those). The rule is OFF for an agent that has produced
+ * neither a telemetry sample nor a single hook event: with no channel that
+ * could ever show a turn, "no activity" is not evidence of anything, and a
+ * Codex/Gemini/grok worker would otherwise read as stalled forever and be
+ * nudged every cooldown while working — the repeated nudging #368 removed. */
export const WORKER_WAKE_STALL_MS = 90_000;
/** Minimum age of pending mail before a held worker is reported in the log. */
export const WORKER_WAKE_REPORT_MS = 60_000;
@@ -81,6 +90,27 @@ export function classifyHook(event: string | undefined, message: string | undefi
return null;
}
+/** A hook event that proves the CLI took a turn — the activity signal every
+ * engine the harness shims produces (Codex, Gemini, grok, … are mapped onto
+ * these names in hive.ts), unlike telemetry, which only Claude Code exports.
+ * SessionStart is the CLI coming up, not a turn: a worker whose boot nudge was
+ * lost has exactly that and nothing else. Notification is the CLI waiting. */
+export function isTurnHook(event: string | undefined): boolean {
+ switch (event) {
+ case 'UserPromptSubmit':
+ case 'PreToolUse':
+ case 'PostToolUse':
+ case 'PostToolUseFailure':
+ case 'Stop':
+ case 'StopFailure':
+ case 'SubagentStart':
+ case 'SubagentStop':
+ return true;
+ default:
+ return false;
+ }
+}
+
/** One worker's live facts, gathered by the caller each beat. */
export interface WorkerWakeFacts {
/** Worker agent id (god is never a candidate). */
@@ -100,6 +130,11 @@ export interface WorkerWakeFacts {
/** Timestamp of the agent's last telemetry usage sample — the CLI's own
* evidence of a turn — or 0/undefined when it has never reported one. */
lastActivityAt?: number;
+ /** True when the telemetry collector holds ANY usage sample for the agent
+ * (even the zero-token one stamped at session start): its CLI exports
+ * telemetry, so a missing turn there means something. Only Claude Code
+ * does; for every other engine the hooks are the activity channel. */
+ hasTelemetry?: boolean;
/** created_at of the OLDEST undrained inbox message, or 0/undefined when
* unknown (the stall rule then stays off — fail closed, as before). */
oldestMailAt?: number;
@@ -160,23 +195,50 @@ export class WorkerWakeWatchdog {
private announcedInboxIds = new Map<string, Set<string>>();
/** agentId → timestamp of the last needsHuman hook notification. */
private lastHumanNeedsAt = new Map<string, number>();
+ /** agentId → timestamp of its last hook event of ANY kind: the agent's hooks
+ * are alive, so a missing turn hook means something. */
+ private hookSeenAt = new Map<string, number>();
+ /** agentId → timestamp of its last hook event that proves a turn. */
+ private lastTurnHookAt = new Map<string, number>();
/** Record a PTY spawn so its boot sequence is left alone. */
noteSpawn(ptyId: string, at = Date.now()): void {
this.spawnedAt.set(ptyId, at);
}
- /** Feed hook events (from HookServer) so a HITL prompt blocks nudges. */
+ /** Feed hook events (from HookServer): a HITL prompt blocks nudges, and any
+ * turn-proving event is activity the stall rule credits — the one channel
+ * every engine has, telemetry being Claude-only. */
noteHook(agentId: string | undefined, event: string | undefined, message: string | undefined, at = Date.now()): void {
if (!agentId) return;
+ this.hookSeenAt.set(agentId, at);
+ if (isTurnHook(event) && at > (this.lastTurnHookAt.get(agentId) ?? 0)) this.lastTurnHookAt.set(agentId, at);
if (classifyHook(event, message) === 'needsHuman') this.lastHumanNeedsAt.set(agentId, at);
}
+ /** When the agent's hooks last proved a turn, or 0 when they never have. */
+ turnHookAt(agentId: string): number {
+ return this.lastTurnHookAt.get(agentId) ?? 0;
+ }
+
+ /** The stall rule on everything known: the beat's telemetry evidence plus
+ * the hooks' — and OFF for an agent that has produced neither a telemetry
+ * sample nor a single hook event, because such an agent cannot show a turn
+ * even when it takes one (see WORKER_WAKE_STALL_MS). */
+ private isStalled(f: WorkerWakeFacts, now: number): boolean {
+ const observable = !!f.hasTelemetry || this.hookSeenAt.has(f.agentId);
+ if (!observable) return false;
+ const lastActivityAt = Math.max(f.lastActivityAt ?? 0, this.turnHookAt(f.agentId));
+ return isStalledWorker({ ...f, lastActivityAt }, now);
+ }
+
/** Forget per-agent state (e.g. the agent's PTY was closed). */
forget(agentId: string, ptyId?: string): void {
this.lastNudgeAt.delete(agentId);
this.announcedInboxIds.delete(agentId);
this.lastHumanNeedsAt.delete(agentId);
+ this.hookSeenAt.delete(agentId);
+ this.lastTurnHookAt.delete(agentId);
this.lastHoldReportAt.delete(agentId);
if (ptyId) this.spawnedAt.delete(ptyId);
}
@@ -196,7 +258,7 @@ export class WorkerWakeWatchdog {
if (f.autoDeliveryPaused) return 'delivery-paused';
if (f.paused) return 'paused';
if (f.halted) return 'halted';
- const stalled = isStalledWorker(f, now);
+ const stalled = this.isStalled(f, now);
if (f.lastOutputAt <= 0 && !stalled) return 'booting'; // never produced output → still booting
if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS && !stalled) return 'mid-turn';
const spawned = this.spawnedAt.get(f.ptyId) ?? 0;
diff --git a/test/worker-wake-stall.test.cjs b/test/worker-wake-stall.test.cjs
index 9e8e652c..ef27b270 100644
--- a/test/worker-wake-stall.test.cjs
+++ b/test/worker-wake-stall.test.cjs
@@ -18,6 +18,7 @@ const {
WorkerWakeWatchdog,
isStalledWorker,
activityEvidenceAt,
+ isTurnHook,
WORKER_WAKE_IDLE_MS,
WORKER_WAKE_STALL_MS,
WORKER_WAKE_COOLDOWN_MS,
@@ -26,8 +27,9 @@ const {
const NOW = 10_000_000; // far enough from epoch that "20 minutes ago" stays positive
-/** A worker whose terminal is chatty (output 1s ago) — the shape that used to
- * read as "mid-turn" forever. */
+/** A Claude worker whose terminal is chatty (output 1s ago) — the shape that
+ * used to read as "mid-turn" forever. Claude Code exports telemetry, so the
+ * collector holds a sample for it (the zero-token boot one at least). */
function chatty(overrides = {}) {
return {
agentId: 'stanley',
@@ -37,10 +39,18 @@ function chatty(overrides = {}) {
autoDeliveryPaused: false,
paused: false,
halted: false,
+ hasTelemetry: true,
...overrides
};
}
+/** The same worker on an engine that exports no telemetry (Codex, Gemini,
+ * grok, …): the collector never sees a sample, so `lastActivityAt` is always
+ * 0 and the hooks are the only activity channel. */
+function codex(overrides = {}) {
+ return chatty({ agentId: 'kevin', ptyId: 'pty-kevin', hasTelemetry: false, lastActivityAt: 0, ...overrides });
+}
+
function watchdog() {
const w = new WorkerWakeWatchdog();
w.noteSpawn('pty-stanley', NOW - 20 * 60_000); // spawned 20 minutes ago
@@ -118,6 +128,78 @@ test('mail younger than the stall window keeps the original quiet-output rule',
assert.equal(w.explain(quiet, NOW), null);
});
+test('isTurnHook: prompt submits, tool boundaries and stops prove a turn; boot, idle and compaction do not', () => {
+ for (const e of ['UserPromptSubmit', 'PreToolUse', 'PostToolUse', 'PostToolUseFailure', 'Stop', 'StopFailure', 'SubagentStart', 'SubagentStop']) {
+ assert.equal(isTurnHook(e), true, e);
+ }
+ for (const e of ['SessionStart', 'SessionEnd', 'Notification', 'PreCompact', 'PostCompact', 'PermissionDenied', 'Unknown', undefined]) {
+ assert.equal(isTurnHook(e), false, String(e));
+ }
+});
+
+// Activity evidence used to come from telemetry alone, and only Claude Code
+// exports it. For a Codex / Gemini / grok worker it was always 0, so once its
+// mail was 90 s old the stall rule fired on every tick and the worker was nudged
+// every cooldown WHILE WORKING, until the file moved to .done — the repeated
+// nudging #368 removed. Now the hooks (which every shimmed engine sends) are
+// evidence too, and an agent with neither channel is never called stalled.
+test('an engine with no telemetry and no hook event is never stalled: the pre-stall rules apply unchanged (fail closed, #368)', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);
+ const working = codex({ oldestMailAt: NOW - 10 * 60_000 });
+ assert.equal(w.explain(working, NOW), 'mid-turn', 'its chatty terminal is all we have, and it says busy');
+ assert.deepEqual(w.decide([working], NOW), []);
+ const quiet = codex({ oldestMailAt: NOW - 10 * 60_000, lastOutputAt: NOW - WORKER_WAKE_IDLE_MS - 1 });
+ assert.deepEqual(w.decide([quiet], NOW), ['kevin'], 'a quiet terminal is nudged once, as before');
+ assert.equal(w.explain(quiet, NOW + WORKER_WAKE_COOLDOWN_MS + 1), 'announced', 'and never again for the same mail — no stall override without evidence');
+});
+
+test('a hook-only engine whose hooks show no turn since the mail landed is stalled and nudged every cooldown', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);
+ const mailAt = NOW - 10 * 60_000;
+ // Its hooks are alive (the CLI came up after the mail) but never proved a turn.
+ w.noteHook('kevin', 'SessionStart', undefined, mailAt + 1_000);
+ w.noteHook('kevin', 'Notification', 'Codex is waiting for your input', mailAt + 2_000);
+ const f = codex({ oldestMailAt: mailAt });
+ assert.equal(w.explain(f, NOW), null, 'SessionStart and an idle Notification are not turns');
+ assert.deepEqual(w.decide([f], NOW), ['kevin']);
+ assert.deepEqual(w.decide([f], NOW + WORKER_WAKE_COOLDOWN_MS), ['kevin'], 'retried every cooldown until it acts');
+ // A turn hook BEFORE the mail does not count either.
+ const w2 = new WorkerWakeWatchdog();
+ w2.noteSpawn('pty-kevin', NOW - 20 * 60_000);
+ w2.noteHook('kevin', 'PostToolUse', undefined, mailAt - 1);
+ assert.equal(w2.explain(f, NOW), null);
+});
+
+test('a turn hook after the mail is activity: the worker is working it, not stalled (no repeated nudging)', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);
+ const mailAt = NOW - 10 * 60_000;
+ w.noteHook('kevin', 'PreToolUse', undefined, NOW - 5_000);
+ const f = codex({ oldestMailAt: mailAt });
+ assert.equal(w.explain(f, NOW), 'mid-turn');
+ assert.deepEqual(w.decide([f], NOW), []);
+ assert.equal(w.turnHookAt('kevin'), NOW - 5_000, 'the beat can log when the hooks last proved a turn');
+ // Hook evidence also covers a Claude worker whose telemetry is dead.
+ const c = new WorkerWakeWatchdog();
+ c.noteSpawn('pty-stanley', NOW - 20 * 60_000);
+ c.noteHook('stanley', 'UserPromptSubmit', undefined, NOW - 3_000);
+ assert.equal(c.explain(chatty({ oldestMailAt: mailAt, lastActivityAt: 0 }), NOW), 'mid-turn');
+});
+
+test('forget() drops the hook memory: a re-spawned agent starts unobservable again', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);
+ w.noteHook('kevin', 'SessionStart', undefined, NOW - 9 * 60_000);
+ const f = codex({ oldestMailAt: NOW - 10 * 60_000 });
+ assert.equal(w.explain(f, NOW), null, 'stalled while its hooks are known');
+ w.forget('kevin', 'pty-kevin');
+ w.noteSpawn('pty-kevin', NOW - 20 * 60_000);
+ assert.equal(w.explain(f, NOW), 'mid-turn', 'no hook seen this session → the stall rule is off');
+ assert.equal(w.turnHookAt('kevin'), 0);
+});
+
test('facts without the new fields behave exactly as before (fail closed)', () => {
const w = watchdog();
assert.equal(w.explain(chatty(), NOW), 'mid-turn');