Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

fix(worker-wake): nudge a stalled worker whatever its terminal prints, and log why one is held

Gastón Péchieued77f6b68560
diff --git a/src/main/index.ts b/src/main/index.ts
index 40634ac6..5b9a0511 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -64,7 +64,7 @@ import { validateBaseUrl, buildAuthHeaders, resolveUpstreamUrl, secretRefFor, IN
import { RosterStore } from './roster';
import { buildWorkerLaunch } from './workerLaunch';
import { ControlRegistry } from './control';
-import { WorkerWakeWatchdog, type WorkerWakeFacts } from './workerWake';
+import { WorkerWakeWatchdog, WORKER_WAKE_REPORT_MS, type WorkerWakeFacts } from './workerWake';
import { inboxNudgeText } from '../shared/hiveNudge';
import { resolveGodName } from '../shared/godIdentity';
import { fetchHireManifest, readHireManifestFiles } from './hire';
@@ -5147,18 +5147,29 @@ function runWorkerWakeBeat(): void {
const ptyId = ptyForAgent(agentId);
if (!ptyId) continue;
const snap = control.snapshot(agentId);
+ const mail = hive.inbox(agentId);
+ // Oldest pending message: the stall rule measures how long the worker has
+ // ignored its mail, and telemetry says whether it has done ANY turn since.
+ let oldestMailAt = 0;
+ for (const m of mail) {
+ const t = Date.parse(m.created_at ?? '');
+ if (Number.isFinite(t) && (oldestMailAt === 0 || t < oldestMailAt)) oldestMailAt = t;
+ }
facts.push({
agentId,
isGod: agentId === reg.godId,
ptyId,
lastOutputAt: ptyManager.lastOutputAt(ptyId) ?? 0,
- inboxIds: hive.inbox(agentId).map((message) => message.id).filter(Boolean),
+ inboxIds: mail.map((message) => message.id).filter(Boolean),
autoDeliveryPaused: snap.autoDeliveryPaused,
paused: snap.paused,
- halted: snap.halted
+ halted: snap.halted,
+ lastActivityAt: telemetry.getAgentUsage(agentId)?.ts ?? 0,
+ oldestMailAt
});
}
- for (const agentId of workerWake.decide(facts, now)) {
+ const nudged = new Set(workerWake.decide(facts, now));
+ for (const agentId of nudged) {
const ptyId = ptyForAgent(agentId);
if (!ptyId) continue;
// Re-read at delivery time, not from the facts snapshot: the agent may have
@@ -5169,6 +5180,19 @@ function runWorkerWakeBeat(): void {
console.log(`[worker-wake] nudging ${agentId} on ${ptyId} (${ids.length} pending)`);
nudgeWorker(ptyId, ids);
}
+ // A worker sitting on old mail without a nudge is the failure this watchdog
+ // exists for — say WHY it is being held, once per cooldown, so the log can
+ // never again read "nothing happened" while a worker starves on its inbox.
+ for (const f of facts) {
+ if (nudged.has(f.agentId) || f.inboxIds.length === 0) continue;
+ const mailAge = f.oldestMailAt && f.oldestMailAt > 0 ? now - f.oldestMailAt : 0;
+ if (mailAge < WORKER_WAKE_REPORT_MS) continue;
+ if (!workerWake.shouldReportHold(f.agentId, now)) continue;
+ const hold = workerWake.explain(f, now);
+ const quiet = f.lastOutputAt > 0 ? `${Math.round((now - f.lastOutputAt) / 1000)}s` : 'never';
+ const active = f.lastActivityAt && f.lastActivityAt > 0 ? `${Math.round((now - f.lastActivityAt) / 1000)}s ago` : 'never';
+ console.warn(`[worker-wake] holding ${f.agentId}: ${hold} (mail pending ${Math.round(mailAge / 1000)}s, pty quiet ${quiet}, last activity ${active})`);
+ }
}
/** (Re)arm the always-on beats (decoupled from the optional heartbeat): the live
diff --git a/src/main/workerWake.ts b/src/main/workerWake.ts
index 0db11bf9..d31bbd75 100644
--- a/src/main/workerWake.ts
+++ b/src/main/workerWake.ts
@@ -44,6 +44,20 @@ export const WORKER_WAKE_BOOT_GRACE_MS = 35_000;
export const WORKER_WAKE_COOLDOWN_MS = 60_000;
/** A permission/HITL notification blocks nudges for this long after it fires. */
export const WORKER_WAKE_HITL_REARM_MS = 5 * 60_000;
+/** Mail this old with NO session activity since it landed = a STALLED worker:
+ * its CLI never took the first turn (a boot-time nudge lost while the TUI was
+ * still drawing, an occluded renderer that never typed one). PTY output cannot
+ * vouch for such a worker — a TUI redraws its chrome without doing any work,
+ * and the boot sequence itself is output — so past this age the quiet-output
+ * and never-output rules are bypassed, and so is the announced-ids edge trigger
+ * (#358 is for a worker that HEARD the announcement; a stalled one did not),
+ * still subject to paused/halted/HITL/boot-grace/cooldown. Observed live
+ * 2026-09-06: a worker sat 17 minutes on its work order with 0 tokens and no
+ * transcript until the human typed "read your inbox" by hand; this watchdog
+ * never fired. */
+export const WORKER_WAKE_STALL_MS = 90_000;
+/** Minimum age of pending mail before a held worker is reported in the log. */
+export const WORKER_WAKE_REPORT_MS = 60_000;
/** A hook event message that means "the agent needs the human" — permission /
* approve / confirm prompts (mirrors the renderer's needsHuman detection in
@@ -83,6 +97,33 @@ export interface WorkerWakeFacts {
autoDeliveryPaused: boolean;
paused: boolean;
halted: boolean;
+ /** Timestamp of the agent's last telemetry usage sample — the CLI's own
+ * evidence of a turn — or 0/undefined when it has never reported one. */
+ lastActivityAt?: number;
+ /** created_at of the OLDEST undrained inbox message, or 0/undefined when
+ * unknown (the stall rule then stays off — fail closed, as before). */
+ oldestMailAt?: number;
+}
+
+/** Why a worker with pending mail is NOT being nudged right now. */
+export type WorkerWakeHold =
+ | 'god' | 'no-mail' | 'no-pty'
+ | 'delivery-paused' | 'paused' | 'halted'
+ | 'booting' | 'mid-turn' | 'boot-grace' | 'hitl' | 'announced' | 'cooldown';
+
+/** The inbox ids that count as mail: non-empty strings only. */
+function liveInboxIds(f: WorkerWakeFacts): Set<string> {
+ return new Set(f.inboxIds.filter((id) => typeof id === 'string' && id.length > 0));
+}
+
+/** Mail has waited WORKER_WAKE_STALL_MS and the CLI has shown no session
+ * activity since it landed: whatever its terminal is printing, this worker is
+ * not working the mail. */
+export function isStalledWorker(f: WorkerWakeFacts, now = Date.now()): boolean {
+ const mailAt = f.oldestMailAt ?? 0;
+ if (mailAt <= 0 || liveInboxIds(f).size === 0) return false;
+ if (now - mailAt < WORKER_WAKE_STALL_MS) return false;
+ return (f.lastActivityAt ?? 0) < mailAt;
}
export class WorkerWakeWatchdog {
@@ -112,33 +153,50 @@ export class WorkerWakeWatchdog {
this.lastNudgeAt.delete(agentId);
this.announcedInboxIds.delete(agentId);
this.lastHumanNeedsAt.delete(agentId);
+ this.lastHoldReportAt.delete(agentId);
if (ptyId) this.spawnedAt.delete(ptyId);
}
/** The worker ids that should be nudged right now, in stable registry order.
* Pure decision — the caller types the nudge. */
+ /** Why this worker is held right now, or null when it should be nudged.
+ * The same checks decide() applies, in the same order, exposed so the beat
+ * can LOG why a worker with old pending mail is not being woken — the
+ * watchdog's silence used to be indistinguishable from "nothing to do".
+ * Pure: never touches the announcement / cooldown memory. */
+ explain(f: WorkerWakeFacts, now = Date.now()): WorkerWakeHold | null {
+ const inboxIds = liveInboxIds(f);
+ if (inboxIds.size === 0) return 'no-mail';
+ if (f.isGod) return 'god';
+ if (!f.ptyId) return 'no-pty';
+ if (f.autoDeliveryPaused) return 'delivery-paused';
+ if (f.paused) return 'paused';
+ if (f.halted) return 'halted';
+ const stalled = isStalledWorker(f, now);
+ if (f.lastOutputAt <= 0 && !stalled) return 'booting'; // never produced output → still booting
+ if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS && !stalled) return 'mid-turn';
+ const spawned = this.spawnedAt.get(f.ptyId) ?? 0;
+ if (spawned > 0 && now - spawned < WORKER_WAKE_BOOT_GRACE_MS) return 'boot-grace';
+ const lastHuman = this.lastHumanNeedsAt.get(f.agentId) ?? 0;
+ if (lastHuman > 0 && now - lastHuman < WORKER_WAKE_HITL_REARM_MS) return 'hitl';
+ // Edge trigger (#358): mail already announced is not announced again — unless
+ // the worker is stalled, i.e. it demonstrably never acted on the announcement.
+ const announced = this.announcedInboxIds.get(f.agentId);
+ if (announced && !stalled && !Array.from(inboxIds).some((id) => !announced.has(id))) return 'announced';
+ const lastNudge = this.lastNudgeAt.get(f.agentId) ?? 0;
+ if (lastNudge > 0 && now - lastNudge < WORKER_WAKE_COOLDOWN_MS) return 'cooldown';
+ return null;
+ }
+
decide(facts: readonly WorkerWakeFacts[], now = Date.now()): string[] {
const out: string[] = [];
for (const f of facts) {
- const inboxIds = new Set(f.inboxIds.filter((id) => typeof id === 'string' && id.length > 0));
+ const inboxIds = liveInboxIds(f);
if (inboxIds.size === 0) {
- // A fully drained inbox starts a fresh announcement cycle and bounds the
- // remembered set even for a worker that lives for months.
this.announcedInboxIds.delete(f.agentId);
continue;
}
- if (f.isGod || !f.ptyId) continue;
- if (f.autoDeliveryPaused || f.paused || f.halted) continue;
- if (f.lastOutputAt <= 0) continue; // never produced output → still booting
- if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS) continue; // mid-turn
- const spawned = this.spawnedAt.get(f.ptyId) ?? 0;
- if (spawned > 0 && now - spawned < WORKER_WAKE_BOOT_GRACE_MS) continue;
- const lastHuman = this.lastHumanNeedsAt.get(f.agentId) ?? 0;
- if (lastHuman > 0 && now - lastHuman < WORKER_WAKE_HITL_REARM_MS) continue;
- const announced = this.announcedInboxIds.get(f.agentId);
- if (announced && !Array.from(inboxIds).some((id) => !announced.has(id))) continue;
- const lastNudge = this.lastNudgeAt.get(f.agentId) ?? 0;
- if (lastNudge > 0 && now - lastNudge < WORKER_WAKE_COOLDOWN_MS) continue;
+ if (this.explain(f, now) !== null) continue;
this.lastNudgeAt.set(f.agentId, now);
this.announcedInboxIds.set(f.agentId, inboxIds);
out.push(f.agentId);
@@ -146,7 +204,18 @@ export class WorkerWakeWatchdog {
return out;
}
- /** Last time this worker was nudged (0 = never) — useful for diagnostics. */
+ /** agentId → when its hold was last reported, so the beat logs a held worker
+ * once per cooldown instead of every 15 s. */
+ private lastHoldReportAt = new Map<string, number>();
+
+ /** True once per WORKER_WAKE_COOLDOWN_MS per worker — the beat's log gate. */
+ shouldReportHold(agentId: string, now = Date.now()): boolean {
+ const last = this.lastHoldReportAt.get(agentId) ?? 0;
+ if (last > 0 && now - last < WORKER_WAKE_COOLDOWN_MS) return false;
+ this.lastHoldReportAt.set(agentId, now);
+ return true;
+ }
+
lastNudge(agentId: string): number {
return this.lastNudgeAt.get(agentId) ?? 0;
}
diff --git a/test/worker-wake-stall.test.cjs b/test/worker-wake-stall.test.cjs
new file mode 100644
index 00000000..fd0f2c8c
--- /dev/null
+++ b/test/worker-wake-stall.test.cjs
@@ -0,0 +1,140 @@
+'use strict';
+// The watchdog's blind spot: a worker that never took its first turn.
+//
+// Observed live 2026-09-06: a god-spawned worker sat 17 minutes on its work
+// order — 0 tokens, no transcript, mail unread — and the watchdog never fired.
+// Its rules infer "busy" from PTY output and "booting" from no output, but a
+// TUI redraws its chrome without doing any work and the boot sequence itself is
+// output. Telemetry is the CLI's own evidence of a turn: mail older than
+// WORKER_WAKE_STALL_MS with no usage sample since it landed is a stalled
+// worker, and the nudge goes in whatever the terminal is printing — and again
+// after each cooldown, even though its ids were already announced (#358's
+// edge trigger is for a worker that heard the announcement; a stalled one did
+// not).
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const loadTs = require('./load-ts.cjs');
+const {
+ WorkerWakeWatchdog,
+ isStalledWorker,
+ WORKER_WAKE_IDLE_MS,
+ WORKER_WAKE_STALL_MS,
+ WORKER_WAKE_COOLDOWN_MS,
+ WORKER_WAKE_HITL_REARM_MS
+} = loadTs('src/main/workerWake.ts');
+
+const NOW = 10_000_000; // far enough from epoch that "20 minutes ago" stays positive
+
+/** A worker whose terminal is chatty (output 1s ago) — the shape that used to
+ * read as "mid-turn" forever. */
+function chatty(overrides = {}) {
+ return {
+ agentId: 'stanley',
+ ptyId: 'pty-stanley',
+ lastOutputAt: NOW - 1_000,
+ inboxIds: ['mail-1'],
+ autoDeliveryPaused: false,
+ paused: false,
+ halted: false,
+ ...overrides
+ };
+}
+
+function watchdog() {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-stanley', NOW - 20 * 60_000); // spawned 20 minutes ago
+ return w;
+}
+
+test('isStalledWorker: old mail with no activity since it landed', () => {
+ const mailAt = NOW - WORKER_WAKE_STALL_MS;
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: 0 }), NOW), true);
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: mailAt - 1 }), NOW), true, 'activity BEFORE the mail does not count');
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: mailAt + 1 }), NOW), false, 'a turn after the mail = working it');
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt + 1, lastActivityAt: 0 }), NOW), false, 'younger than the stall window');
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: undefined, lastActivityAt: 0 }), NOW), false, 'unknown mail age → rule off (fail closed)');
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: 0, lastActivityAt: 0 }), NOW), false);
+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: 0, inboxIds: [] }), NOW), false, 'no mail, nothing to stall on');
+});
+
+test('a stalled worker is nudged even though its terminal is chatty (the 17-minute case)', () => {
+ const w = watchdog();
+ const f = chatty({ oldestMailAt: NOW - 17 * 60_000, lastActivityAt: 0 });
+ assert.equal(w.explain(f, NOW), null);
+ assert.deepEqual(w.decide([f], NOW), ['stanley']);
+});
+
+test('a stalled worker that NEVER produced output is nudged too (boot never happened)', () => {
+ const w = watchdog();
+ const f = chatty({ lastOutputAt: 0, oldestMailAt: NOW - 2 * 60_000, lastActivityAt: 0 });
+ assert.deepEqual(w.decide([f], NOW), ['stanley']);
+});
+
+test('a stalled worker is nudged AGAIN after the cooldown although its mail ids were already announced', () => {
+ const w = watchdog();
+ const f = chatty({ oldestMailAt: NOW - 10 * 60_000, lastActivityAt: 0 });
+ assert.deepEqual(w.decide([f], NOW), ['stanley']);
+ assert.equal(w.explain(f, NOW + WORKER_WAKE_COOLDOWN_MS - 1), 'cooldown');
+ assert.deepEqual(w.decide([f], NOW + WORKER_WAKE_COOLDOWN_MS), ['stanley'], 'retries every cooldown until the mail drains');
+ // Once the CLI shows a turn after the mail, the edge trigger rules again:
+ // same ids, already announced → held, exactly as #358 intends.
+ const awake = chatty({ oldestMailAt: NOW - 10 * 60_000, lastActivityAt: NOW + WORKER_WAKE_COOLDOWN_MS + 5 });
+ assert.equal(w.explain(awake, NOW + 2 * WORKER_WAKE_COOLDOWN_MS + WORKER_WAKE_IDLE_MS + 10), 'announced');
+});
+
+test('a chatty worker with a turn since the mail landed is mid-turn, not stalled', () => {
+ const w = watchdog();
+ const f = chatty({ oldestMailAt: NOW - 17 * 60_000, lastActivityAt: NOW - 5_000 });
+ assert.equal(w.explain(f, NOW), 'mid-turn');
+ assert.deepEqual(w.decide([f], NOW), []);
+});
+
+test('mail younger than the stall window keeps the original quiet-output rule', () => {
+ const w = watchdog();
+ const young = chatty({ oldestMailAt: NOW - 30_000, lastActivityAt: 0 });
+ assert.equal(w.explain(young, NOW), 'mid-turn');
+ const quiet = chatty({ oldestMailAt: NOW - 30_000, lastActivityAt: 0, lastOutputAt: NOW - WORKER_WAKE_IDLE_MS - 1 });
+ assert.equal(w.explain(quiet, NOW), null);
+});
+
+test('facts without the new fields behave exactly as before (fail closed)', () => {
+ const w = watchdog();
+ assert.equal(w.explain(chatty(), NOW), 'mid-turn');
+ assert.equal(w.explain(chatty({ lastOutputAt: 0 }), NOW), 'booting');
+ const quiet = chatty({ lastOutputAt: NOW - WORKER_WAKE_IDLE_MS - 1 });
+ assert.deepEqual(w.decide([quiet], NOW), ['stanley']);
+ assert.equal(w.explain(quiet, NOW + WORKER_WAKE_COOLDOWN_MS + 1), 'announced', 'same mail is not re-announced (#358)');
+});
+
+test('the stall rule never overrides paused / halted / HITL / cooldown / boot grace', () => {
+ const stalled = { oldestMailAt: NOW - 10 * 60_000, lastActivityAt: 0 };
+ assert.equal(watchdog().explain(chatty({ ...stalled, paused: true }), NOW), 'paused');
+ assert.equal(watchdog().explain(chatty({ ...stalled, halted: true }), NOW), 'halted');
+ assert.equal(watchdog().explain(chatty({ ...stalled, autoDeliveryPaused: true }), NOW), 'delivery-paused');
+
+ const hitl = watchdog();
+ hitl.noteHook('stanley', 'Notification', 'Claude needs your permission to run Bash', NOW - WORKER_WAKE_HITL_REARM_MS + 1);
+ assert.equal(hitl.explain(chatty(stalled), NOW), 'hitl');
+
+ const fresh = new WorkerWakeWatchdog();
+ fresh.noteSpawn('pty-stanley', NOW - 10_000);
+ assert.equal(fresh.explain(chatty(stalled), NOW), 'boot-grace');
+});
+
+test('explain names every hold so the beat can log why a worker starves', () => {
+ const w = watchdog();
+ assert.equal(w.explain(chatty({ isGod: true }), NOW), 'god');
+ assert.equal(w.explain(chatty({ inboxIds: [] }), NOW), 'no-mail');
+ assert.equal(w.explain(chatty({ inboxIds: ['', 42] }), NOW), 'no-mail', 'junk ids do not count as mail');
+ assert.equal(w.explain(chatty({ ptyId: undefined }), NOW), 'no-pty');
+});
+
+test('a hold is reported once per cooldown per worker, and forget() resets it', () => {
+ const w = watchdog();
+ assert.equal(w.shouldReportHold('stanley', NOW), true);
+ assert.equal(w.shouldReportHold('stanley', NOW + 1_000), false);
+ assert.equal(w.shouldReportHold('stanley', NOW + WORKER_WAKE_COOLDOWN_MS), true);
+ assert.equal(w.shouldReportHold('other', NOW + 2_000), true, 'per worker');
+ w.forget('stanley', 'pty-stanley');
+ assert.equal(w.shouldReportHold('stanley', NOW + WORKER_WAKE_COOLDOWN_MS + 1_000), true);
+});