an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
fix(worker-wake): nudge a stalled worker whatever its terminal prints, and log why one is held
Gastón Péchieued77f6b68560
diff --git a/src/main/index.ts b/src/main/index.tsindex 40634ac6..5b9a0511 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -64,7 +64,7 @@ import { validateBaseUrl, buildAuthHeaders, resolveUpstreamUrl, secretRefFor, INimport { RosterStore } from './roster';import { buildWorkerLaunch } from './workerLaunch';import { ControlRegistry } from './control';-import { WorkerWakeWatchdog, type WorkerWakeFacts } from './workerWake';+import { WorkerWakeWatchdog, WORKER_WAKE_REPORT_MS, type WorkerWakeFacts } from './workerWake';import { inboxNudgeText } from '../shared/hiveNudge';import { resolveGodName } from '../shared/godIdentity';import { fetchHireManifest, readHireManifestFiles } from './hire';@@ -5147,18 +5147,29 @@ function runWorkerWakeBeat(): void {const ptyId = ptyForAgent(agentId);if (!ptyId) continue;const snap = control.snapshot(agentId);+ const mail = hive.inbox(agentId);+ // Oldest pending message: the stall rule measures how long the worker has+ // ignored its mail, and telemetry says whether it has done ANY turn since.+ let oldestMailAt = 0;+ for (const m of mail) {+ const t = Date.parse(m.created_at ?? '');+ if (Number.isFinite(t) && (oldestMailAt === 0 || t < oldestMailAt)) oldestMailAt = t;+ }facts.push({agentId,isGod: agentId === reg.godId,ptyId,lastOutputAt: ptyManager.lastOutputAt(ptyId) ?? 0,- inboxIds: hive.inbox(agentId).map((message) => message.id).filter(Boolean),+ inboxIds: mail.map((message) => message.id).filter(Boolean),autoDeliveryPaused: snap.autoDeliveryPaused,paused: snap.paused,- halted: snap.halted+ halted: snap.halted,+ lastActivityAt: telemetry.getAgentUsage(agentId)?.ts ?? 0,+ oldestMailAt});}- for (const agentId of workerWake.decide(facts, now)) {+ const nudged = new Set(workerWake.decide(facts, now));+ for (const agentId of nudged) {const ptyId = ptyForAgent(agentId);if (!ptyId) continue;// Re-read at delivery time, not from the facts snapshot: the agent may have@@ -5169,6 +5180,19 @@ function runWorkerWakeBeat(): void {console.log(`[worker-wake] nudging ${agentId} on ${ptyId} (${ids.length} pending)`);nudgeWorker(ptyId, ids);}+ // A worker sitting on old mail without a nudge is the failure this watchdog+ // exists for — say WHY it is being held, once per cooldown, so the log can+ // never again read "nothing happened" while a worker starves on its inbox.+ for (const f of facts) {+ if (nudged.has(f.agentId) || f.inboxIds.length === 0) continue;+ const mailAge = f.oldestMailAt && f.oldestMailAt > 0 ? now - f.oldestMailAt : 0;+ if (mailAge < WORKER_WAKE_REPORT_MS) continue;+ if (!workerWake.shouldReportHold(f.agentId, now)) continue;+ const hold = workerWake.explain(f, now);+ const quiet = f.lastOutputAt > 0 ? `${Math.round((now - f.lastOutputAt) / 1000)}s` : 'never';+ const active = f.lastActivityAt && f.lastActivityAt > 0 ? `${Math.round((now - f.lastActivityAt) / 1000)}s ago` : 'never';+ console.warn(`[worker-wake] holding ${f.agentId}: ${hold} (mail pending ${Math.round(mailAge / 1000)}s, pty quiet ${quiet}, last activity ${active})`);+ }}/** (Re)arm the always-on beats (decoupled from the optional heartbeat): the livediff --git a/src/main/workerWake.ts b/src/main/workerWake.tsindex 0db11bf9..d31bbd75 100644--- a/src/main/workerWake.ts+++ b/src/main/workerWake.ts@@ -44,6 +44,20 @@ export const WORKER_WAKE_BOOT_GRACE_MS = 35_000;export const WORKER_WAKE_COOLDOWN_MS = 60_000;/** A permission/HITL notification blocks nudges for this long after it fires. */export const WORKER_WAKE_HITL_REARM_MS = 5 * 60_000;+/** Mail this old with NO session activity since it landed = a STALLED worker:+ * its CLI never took the first turn (a boot-time nudge lost while the TUI was+ * still drawing, an occluded renderer that never typed one). PTY output cannot+ * vouch for such a worker — a TUI redraws its chrome without doing any work,+ * and the boot sequence itself is output — so past this age the quiet-output+ * and never-output rules are bypassed, and so is the announced-ids edge trigger+ * (#358 is for a worker that HEARD the announcement; a stalled one did not),+ * still subject to paused/halted/HITL/boot-grace/cooldown. Observed live+ * 2026-09-06: a worker sat 17 minutes on its work order with 0 tokens and no+ * transcript until the human typed "read your inbox" by hand; this watchdog+ * never fired. */+export const WORKER_WAKE_STALL_MS = 90_000;+/** Minimum age of pending mail before a held worker is reported in the log. */+export const WORKER_WAKE_REPORT_MS = 60_000;/** A hook event message that means "the agent needs the human" — permission /* approve / confirm prompts (mirrors the renderer's needsHuman detection in@@ -83,6 +97,33 @@ export interface WorkerWakeFacts {autoDeliveryPaused: boolean;paused: boolean;halted: boolean;+ /** Timestamp of the agent's last telemetry usage sample — the CLI's own+ * evidence of a turn — or 0/undefined when it has never reported one. */+ lastActivityAt?: number;+ /** created_at of the OLDEST undrained inbox message, or 0/undefined when+ * unknown (the stall rule then stays off — fail closed, as before). */+ oldestMailAt?: number;+}++/** Why a worker with pending mail is NOT being nudged right now. */+export type WorkerWakeHold =+ | 'god' | 'no-mail' | 'no-pty'+ | 'delivery-paused' | 'paused' | 'halted'+ | 'booting' | 'mid-turn' | 'boot-grace' | 'hitl' | 'announced' | 'cooldown';++/** The inbox ids that count as mail: non-empty strings only. */+function liveInboxIds(f: WorkerWakeFacts): Set<string> {+ return new Set(f.inboxIds.filter((id) => typeof id === 'string' && id.length > 0));+}++/** Mail has waited WORKER_WAKE_STALL_MS and the CLI has shown no session+ * activity since it landed: whatever its terminal is printing, this worker is+ * not working the mail. */+export function isStalledWorker(f: WorkerWakeFacts, now = Date.now()): boolean {+ const mailAt = f.oldestMailAt ?? 0;+ if (mailAt <= 0 || liveInboxIds(f).size === 0) return false;+ if (now - mailAt < WORKER_WAKE_STALL_MS) return false;+ return (f.lastActivityAt ?? 0) < mailAt;}export class WorkerWakeWatchdog {@@ -112,33 +153,50 @@ export class WorkerWakeWatchdog {this.lastNudgeAt.delete(agentId);this.announcedInboxIds.delete(agentId);this.lastHumanNeedsAt.delete(agentId);+ this.lastHoldReportAt.delete(agentId);if (ptyId) this.spawnedAt.delete(ptyId);}/** The worker ids that should be nudged right now, in stable registry order.* Pure decision — the caller types the nudge. */+ /** Why this worker is held right now, or null when it should be nudged.+ * The same checks decide() applies, in the same order, exposed so the beat+ * can LOG why a worker with old pending mail is not being woken — the+ * watchdog's silence used to be indistinguishable from "nothing to do".+ * Pure: never touches the announcement / cooldown memory. */+ explain(f: WorkerWakeFacts, now = Date.now()): WorkerWakeHold | null {+ const inboxIds = liveInboxIds(f);+ if (inboxIds.size === 0) return 'no-mail';+ if (f.isGod) return 'god';+ if (!f.ptyId) return 'no-pty';+ if (f.autoDeliveryPaused) return 'delivery-paused';+ if (f.paused) return 'paused';+ if (f.halted) return 'halted';+ const stalled = isStalledWorker(f, now);+ if (f.lastOutputAt <= 0 && !stalled) return 'booting'; // never produced output → still booting+ if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS && !stalled) return 'mid-turn';+ const spawned = this.spawnedAt.get(f.ptyId) ?? 0;+ if (spawned > 0 && now - spawned < WORKER_WAKE_BOOT_GRACE_MS) return 'boot-grace';+ const lastHuman = this.lastHumanNeedsAt.get(f.agentId) ?? 0;+ if (lastHuman > 0 && now - lastHuman < WORKER_WAKE_HITL_REARM_MS) return 'hitl';+ // Edge trigger (#358): mail already announced is not announced again — unless+ // the worker is stalled, i.e. it demonstrably never acted on the announcement.+ const announced = this.announcedInboxIds.get(f.agentId);+ if (announced && !stalled && !Array.from(inboxIds).some((id) => !announced.has(id))) return 'announced';+ const lastNudge = this.lastNudgeAt.get(f.agentId) ?? 0;+ if (lastNudge > 0 && now - lastNudge < WORKER_WAKE_COOLDOWN_MS) return 'cooldown';+ return null;+ }+decide(facts: readonly WorkerWakeFacts[], now = Date.now()): string[] {const out: string[] = [];for (const f of facts) {- const inboxIds = new Set(f.inboxIds.filter((id) => typeof id === 'string' && id.length > 0));+ const inboxIds = liveInboxIds(f);if (inboxIds.size === 0) {- // A fully drained inbox starts a fresh announcement cycle and bounds the- // remembered set even for a worker that lives for months.this.announcedInboxIds.delete(f.agentId);continue;}- if (f.isGod || !f.ptyId) continue;- if (f.autoDeliveryPaused || f.paused || f.halted) continue;- if (f.lastOutputAt <= 0) continue; // never produced output → still booting- if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS) continue; // mid-turn- const spawned = this.spawnedAt.get(f.ptyId) ?? 0;- if (spawned > 0 && now - spawned < WORKER_WAKE_BOOT_GRACE_MS) continue;- const lastHuman = this.lastHumanNeedsAt.get(f.agentId) ?? 0;- if (lastHuman > 0 && now - lastHuman < WORKER_WAKE_HITL_REARM_MS) continue;- const announced = this.announcedInboxIds.get(f.agentId);- if (announced && !Array.from(inboxIds).some((id) => !announced.has(id))) continue;- const lastNudge = this.lastNudgeAt.get(f.agentId) ?? 0;- if (lastNudge > 0 && now - lastNudge < WORKER_WAKE_COOLDOWN_MS) continue;+ if (this.explain(f, now) !== null) continue;this.lastNudgeAt.set(f.agentId, now);this.announcedInboxIds.set(f.agentId, inboxIds);out.push(f.agentId);@@ -146,7 +204,18 @@ export class WorkerWakeWatchdog {return out;}- /** Last time this worker was nudged (0 = never) — useful for diagnostics. */+ /** agentId → when its hold was last reported, so the beat logs a held worker+ * once per cooldown instead of every 15 s. */+ private lastHoldReportAt = new Map<string, number>();++ /** True once per WORKER_WAKE_COOLDOWN_MS per worker — the beat's log gate. */+ shouldReportHold(agentId: string, now = Date.now()): boolean {+ const last = this.lastHoldReportAt.get(agentId) ?? 0;+ if (last > 0 && now - last < WORKER_WAKE_COOLDOWN_MS) return false;+ this.lastHoldReportAt.set(agentId, now);+ return true;+ }+lastNudge(agentId: string): number {return this.lastNudgeAt.get(agentId) ?? 0;}diff --git a/test/worker-wake-stall.test.cjs b/test/worker-wake-stall.test.cjsnew file mode 100644index 00000000..fd0f2c8c--- /dev/null+++ b/test/worker-wake-stall.test.cjs@@ -0,0 +1,140 @@+'use strict';+// The watchdog's blind spot: a worker that never took its first turn.+//+// Observed live 2026-09-06: a god-spawned worker sat 17 minutes on its work+// order — 0 tokens, no transcript, mail unread — and the watchdog never fired.+// Its rules infer "busy" from PTY output and "booting" from no output, but a+// TUI redraws its chrome without doing any work and the boot sequence itself is+// output. Telemetry is the CLI's own evidence of a turn: mail older than+// WORKER_WAKE_STALL_MS with no usage sample since it landed is a stalled+// worker, and the nudge goes in whatever the terminal is printing — and again+// after each cooldown, even though its ids were already announced (#358's+// edge trigger is for a worker that heard the announcement; a stalled one did+// not).+const { test } = require('node:test');+const assert = require('node:assert/strict');+const loadTs = require('./load-ts.cjs');+const {+ WorkerWakeWatchdog,+ isStalledWorker,+ WORKER_WAKE_IDLE_MS,+ WORKER_WAKE_STALL_MS,+ WORKER_WAKE_COOLDOWN_MS,+ WORKER_WAKE_HITL_REARM_MS+} = loadTs('src/main/workerWake.ts');++const NOW = 10_000_000; // far enough from epoch that "20 minutes ago" stays positive++/** A worker whose terminal is chatty (output 1s ago) — the shape that used to+ * read as "mid-turn" forever. */+function chatty(overrides = {}) {+ return {+ agentId: 'stanley',+ ptyId: 'pty-stanley',+ lastOutputAt: NOW - 1_000,+ inboxIds: ['mail-1'],+ autoDeliveryPaused: false,+ paused: false,+ halted: false,+ ...overrides+ };+}++function watchdog() {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-stanley', NOW - 20 * 60_000); // spawned 20 minutes ago+ return w;+}++test('isStalledWorker: old mail with no activity since it landed', () => {+ const mailAt = NOW - WORKER_WAKE_STALL_MS;+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: 0 }), NOW), true);+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: mailAt - 1 }), NOW), true, 'activity BEFORE the mail does not count');+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: mailAt + 1 }), NOW), false, 'a turn after the mail = working it');+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt + 1, lastActivityAt: 0 }), NOW), false, 'younger than the stall window');+ assert.equal(isStalledWorker(chatty({ oldestMailAt: undefined, lastActivityAt: 0 }), NOW), false, 'unknown mail age → rule off (fail closed)');+ assert.equal(isStalledWorker(chatty({ oldestMailAt: 0, lastActivityAt: 0 }), NOW), false);+ assert.equal(isStalledWorker(chatty({ oldestMailAt: mailAt, lastActivityAt: 0, inboxIds: [] }), NOW), false, 'no mail, nothing to stall on');+});++test('a stalled worker is nudged even though its terminal is chatty (the 17-minute case)', () => {+ const w = watchdog();+ const f = chatty({ oldestMailAt: NOW - 17 * 60_000, lastActivityAt: 0 });+ assert.equal(w.explain(f, NOW), null);+ assert.deepEqual(w.decide([f], NOW), ['stanley']);+});++test('a stalled worker that NEVER produced output is nudged too (boot never happened)', () => {+ const w = watchdog();+ const f = chatty({ lastOutputAt: 0, oldestMailAt: NOW - 2 * 60_000, lastActivityAt: 0 });+ assert.deepEqual(w.decide([f], NOW), ['stanley']);+});++test('a stalled worker is nudged AGAIN after the cooldown although its mail ids were already announced', () => {+ const w = watchdog();+ const f = chatty({ oldestMailAt: NOW - 10 * 60_000, lastActivityAt: 0 });+ assert.deepEqual(w.decide([f], NOW), ['stanley']);+ assert.equal(w.explain(f, NOW + WORKER_WAKE_COOLDOWN_MS - 1), 'cooldown');+ assert.deepEqual(w.decide([f], NOW + WORKER_WAKE_COOLDOWN_MS), ['stanley'], 'retries every cooldown until the mail drains');+ // Once the CLI shows a turn after the mail, the edge trigger rules again:+ // same ids, already announced → held, exactly as #358 intends.+ const awake = chatty({ oldestMailAt: NOW - 10 * 60_000, lastActivityAt: NOW + WORKER_WAKE_COOLDOWN_MS + 5 });+ assert.equal(w.explain(awake, NOW + 2 * WORKER_WAKE_COOLDOWN_MS + WORKER_WAKE_IDLE_MS + 10), 'announced');+});++test('a chatty worker with a turn since the mail landed is mid-turn, not stalled', () => {+ const w = watchdog();+ const f = chatty({ oldestMailAt: NOW - 17 * 60_000, lastActivityAt: NOW - 5_000 });+ assert.equal(w.explain(f, NOW), 'mid-turn');+ assert.deepEqual(w.decide([f], NOW), []);+});++test('mail younger than the stall window keeps the original quiet-output rule', () => {+ const w = watchdog();+ const young = chatty({ oldestMailAt: NOW - 30_000, lastActivityAt: 0 });+ assert.equal(w.explain(young, NOW), 'mid-turn');+ const quiet = chatty({ oldestMailAt: NOW - 30_000, lastActivityAt: 0, lastOutputAt: NOW - WORKER_WAKE_IDLE_MS - 1 });+ assert.equal(w.explain(quiet, NOW), null);+});++test('facts without the new fields behave exactly as before (fail closed)', () => {+ const w = watchdog();+ assert.equal(w.explain(chatty(), NOW), 'mid-turn');+ assert.equal(w.explain(chatty({ lastOutputAt: 0 }), NOW), 'booting');+ const quiet = chatty({ lastOutputAt: NOW - WORKER_WAKE_IDLE_MS - 1 });+ assert.deepEqual(w.decide([quiet], NOW), ['stanley']);+ assert.equal(w.explain(quiet, NOW + WORKER_WAKE_COOLDOWN_MS + 1), 'announced', 'same mail is not re-announced (#358)');+});++test('the stall rule never overrides paused / halted / HITL / cooldown / boot grace', () => {+ const stalled = { oldestMailAt: NOW - 10 * 60_000, lastActivityAt: 0 };+ assert.equal(watchdog().explain(chatty({ ...stalled, paused: true }), NOW), 'paused');+ assert.equal(watchdog().explain(chatty({ ...stalled, halted: true }), NOW), 'halted');+ assert.equal(watchdog().explain(chatty({ ...stalled, autoDeliveryPaused: true }), NOW), 'delivery-paused');++ const hitl = watchdog();+ hitl.noteHook('stanley', 'Notification', 'Claude needs your permission to run Bash', NOW - WORKER_WAKE_HITL_REARM_MS + 1);+ assert.equal(hitl.explain(chatty(stalled), NOW), 'hitl');++ const fresh = new WorkerWakeWatchdog();+ fresh.noteSpawn('pty-stanley', NOW - 10_000);+ assert.equal(fresh.explain(chatty(stalled), NOW), 'boot-grace');+});++test('explain names every hold so the beat can log why a worker starves', () => {+ const w = watchdog();+ assert.equal(w.explain(chatty({ isGod: true }), NOW), 'god');+ assert.equal(w.explain(chatty({ inboxIds: [] }), NOW), 'no-mail');+ assert.equal(w.explain(chatty({ inboxIds: ['', 42] }), NOW), 'no-mail', 'junk ids do not count as mail');+ assert.equal(w.explain(chatty({ ptyId: undefined }), NOW), 'no-pty');+});++test('a hold is reported once per cooldown per worker, and forget() resets it', () => {+ const w = watchdog();+ assert.equal(w.shouldReportHold('stanley', NOW), true);+ assert.equal(w.shouldReportHold('stanley', NOW + 1_000), false);+ assert.equal(w.shouldReportHold('stanley', NOW + WORKER_WAKE_COOLDOWN_MS), true);+ assert.equal(w.shouldReportHold('other', NOW + 2_000), true, 'per worker');+ w.forget('stanley', 'pty-stanley');+ assert.equal(w.shouldReportHold('stanley', NOW + WORKER_WAKE_COOLDOWN_MS + 1_000), true);+});