Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

fix(worker-wake): main-process watchdog wakes idle workers on undrained inbox

Aravind Rao68cbc25cba67
diff --git a/src/main/hooks.ts b/src/main/hooks.ts
index 52195d84..745d0edf 100644
--- a/src/main/hooks.ts
+++ b/src/main/hooks.ts
@@ -67,7 +67,11 @@ export class HookServer {
private control?: ControlRegistry,
/** Circuit breaker (Lane A #6.6b) — fed the hook-derived signals (session id,
* repeated identical tool calls). Optional so the server still runs without it. */
- private breaker?: CircuitBreaker
+ private breaker?: CircuitBreaker,
+ /** Optional observer of every hook boundary (agentId, event, message). The
+ * worker inbox-wake watchdog (workerWake.ts) feeds on this to learn when an
+ * agent is parked on a permission/HITL prompt so it never types into it. */
+ private onEvent?: (agentId: string | undefined, event: string, message: string | undefined) => void
) {}
start(): void {
@@ -115,6 +119,7 @@ export class HookServer {
private handle(p: HookPayload): unknown {
const agentId = p.agent_id ?? undefined;
const event = p.hook_event_name ?? 'Unknown';
+ this.onEvent?.(agentId, event, p.message);
if (agentId && typeof p.transcript_path === 'string' && p.transcript_path) {
this.transcriptPaths.set(agentId, p.transcript_path);
}
diff --git a/src/main/index.ts b/src/main/index.ts
index 45d288e4..a2d06b6a 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -59,6 +59,7 @@ import * as integrations from './integrations';
import { validateBaseUrl, buildAuthHeaders, resolveUpstreamUrl, secretRefFor, INTEGRATION_TEMPLATES } from '../shared/integrations';
import { RosterStore } from './roster';
import { ControlRegistry } from './control';
+import { WorkerWakeWatchdog, WORKER_WAKE_NUDGE, type WorkerWakeFacts } from './workerWake';
import { fetchHireManifest, readHireManifestFile } from './hire';
import { parseHireDeepLink, type HireManifest } from '../shared/hire';
import { ClosingTimeController } from './closingTime';
@@ -262,9 +263,15 @@ let breakerBeatTimer: ReturnType<typeof setInterval> | null = null;
// Feed the breaker's api_error-storm trip from Oscar's OTel api_error spans —
// Jim's one breaker input with no on-branch source (telemetry.onApiError seam).
telemetry.onApiError((agentId) => breaker.recordError(agentId));
+// Worker inbox-wake watchdog (#151): finds idle workers with undrained inbox mail
+// and types the same guarded nudge the renderer would have (so a throttled
+// background window can't leave a worker parked on an unread inbox forever).
+// HookServer feeds it the hook stream so a permission/HITL prompt blocks nudges.
+const workerWake = new WorkerWakeWatchdog();
// HookServer needs BOTH: Oscar's control registry (HITL pause/gate/steer/halt via
// hook returns) AND Jim's breaker (feed recordToolUse on each PostToolUse).
-const hookServer = new HookServer(hive, () => liveWebContents(), () => readConfig(), control, breaker);
+const hookServer = new HookServer(hive, () => liveWebContents(), () => readConfig(), control, breaker,
+ (agentId, event, message) => workerWake.noteHook(agentId, event, message));
const memory = new MemoryManager(
() => readConfig().harnessHome,
() => { const c = readConfig(); return { enabled: c.semanticMemory !== false, model: c.embeddingModel ?? 'minilm' }; }
@@ -398,6 +405,8 @@ function teardownPty(id: string): void {
const agentId = ptyToAgent.get(id);
if (agentId) {
ptyToAgent.delete(id);
+ // Drop watchdog state so a dead agent can't get nudged or leak its grace.
+ try { workerWake.forget(agentId, id); } catch { /* best-effort */ }
// Drop breaker state so a dead agent can't leak/zombie a tripped level.
try { breaker.forget(agentId); } catch { /* best-effort */ }
// W1 — kill this agent's proxy-bridge sidecar (qwen), if any, so a dead
@@ -2681,7 +2690,12 @@ async function spawnAgentCore(opts: AgentSpawnOptions, owner: Electron.WebConten
}
// Remember which agent owns this PTY so closing the tab can archive it. A
// live terminal means active — ensureAgent above already cleared `archived`.
- if (opts.hive?.id) ptyToAgent.set(opts.id, opts.hive.id);
+ if (opts.hive?.id) {
+ ptyToAgent.set(opts.id, opts.hive.id);
+ // Worker inbox-wake watchdog (#151): boot grace starts at spawn so the
+ // initial orientation prompt is never mistaken for an idle agent.
+ workerWake.noteSpawn(opts.id);
+ }
// Pre-accept Claude Code's bypass-mode warning + folder-trust dialog so the
// agent (spawned with --permission-mode bypassPermissions) doesn't stall on an
// interactive prompt it can't answer and exit code 1. Best-effort, never blocks.
@@ -4623,6 +4637,64 @@ function bootstrapHiveServices(): void {
armAlwaysOnBeats();
}
+/** Cadence of the worker inbox-wake watchdog (#151). Well under the renderer's
+ * own nudge cooldown so a throttled window is caught within ~15s of a stall. */
+const WORKER_WAKE_POLL_MS = 15_000;
+let workerWakeTimer: ReturnType<typeof setInterval> | null = null;
+
+/** Type the renderer's guarded nudge into one worker's PTY — text first, Enter a
+ * tick later (the exact submitToPty pattern: a single-chunk write would land the
+ * "\r" inside the input box and never submit). Best-effort + never throws. */
+function nudgeWorker(ptyId: string): void {
+ const wrote = ptyManager.write(ptyId, WORKER_WAKE_NUDGE);
+ if (!wrote.ok) { console.warn(`[worker-wake] write failed for ${ptyId}: ${wrote.error}`); return; }
+ setTimeout(() => {
+ try {
+ const submitted = ptyManager.write(ptyId, '\r');
+ if (!submitted.ok) console.warn(`[worker-wake] submit failed for ${ptyId}: ${submitted.error}`);
+ } catch (e) { console.error('[worker-wake] submit threw:', e); }
+ }, 140);
+}
+
+/** Main-process inbox-wake beat (issue #151, fix A): the renderer's idle nudge
+ * (useHive.ts) is the only path that wakes a worker parked on an undrained
+ * inbox — and it lives on a setInterval in the renderer, which a throttled or
+ * occluded window stops honoring. This beat is the renderer-INDEPENDENT fallback:
+ * it gathers live-worker facts (PTY quiescence, inbox depth, control flags) and
+ * lets WorkerWakeWatchdog.decide apply the exact renderer guards (idle-only,
+ * post-boot-grace, not paused/halted, no pending HITL, cooldown), then types the
+ * same nudge the renderer would have. God is never a candidate (its heartbeat
+ * path already re-engages it). */
+function runWorkerWakeBeat(): void {
+ if (!hive.enabled()) return;
+ const reg = hive.registry();
+ if (!reg?.agents || !reg.godId) return;
+ const now = Date.now();
+ const facts: WorkerWakeFacts[] = [];
+ for (const [agentId, a] of Object.entries(reg.agents)) {
+ if (agentId === reg.godId || a?.archived) continue;
+ const ptyId = ptyForAgent(agentId);
+ if (!ptyId) continue;
+ const snap = control.snapshot(agentId);
+ facts.push({
+ agentId,
+ isGod: agentId === reg.godId,
+ ptyId,
+ lastOutputAt: ptyManager.lastOutputAt(ptyId) ?? 0,
+ inboxCount: hive.inbox(agentId).length,
+ autoDeliveryPaused: snap.autoDeliveryPaused,
+ paused: snap.paused,
+ halted: snap.halted
+ });
+ }
+ for (const agentId of workerWake.decide(facts, now)) {
+ const ptyId = ptyForAgent(agentId);
+ if (!ptyId) continue;
+ console.log(`[worker-wake] nudging ${agentId} on ${ptyId}`);
+ nudgeWorker(ptyId);
+ }
+}
+
/** (Re)arm the always-on beats (decoupled from the optional heartbeat): the live
* fleet snapshot Michael reads (~8s) + the breaker/cost-ledger beat (~30s).
* Guarded (clear-then-set) so a re-bootstrap (changeHome recovery) OR a
@@ -4634,6 +4706,9 @@ function armAlwaysOnBeats(): void {
fleetTimer = setInterval(writeFleetSnapshot, 8_000);
if (breakerBeatTimer) clearInterval(breakerBeatTimer);
breakerBeatTimer = setInterval(() => { try { runBreakerBeat(300_000); } catch (e) { console.error('[breaker beat]', e); } }, 30_000);
+ if (workerWakeTimer) clearInterval(workerWakeTimer);
+ workerWakeTimer = setInterval(() => { try { runWorkerWakeBeat(); } catch (e) { console.error('[worker-wake beat]', e); } }, WORKER_WAKE_POLL_MS);
+ runWorkerWakeBeat(); // catch-up on arm — power-resume re-arms and drains the backlog
}
/** Wall-clock instant we last observed the machine suspend or lock, so a resume
diff --git a/src/main/workerWake.ts b/src/main/workerWake.ts
new file mode 100644
index 00000000..feadcea2
--- /dev/null
+++ b/src/main/workerWake.ts
@@ -0,0 +1,137 @@
+/**
+ * WorkerWakeWatchdog — main-process inbox-wake watchdog for worker agents (#151).
+ *
+ * The renderer's idle inbox-wake nudge (useHive.ts effect #3) is the ONLY wake
+ * path for a worker that has gone quiet at its prompt: it polls on a setInterval
+ * in the renderer, so a throttled/occluded window (Chromium suspends background
+ * setInterval timers) can miss the moment mail lands and the worker then sits on
+ * an undrained inbox forever — the orchestrator ("god") never has this problem
+ * because the main process re-engages it on its own heartbeat cadence.
+ *
+ * This watchdog is the worker-side counterpart: on a cadence it finds live
+ * workers that are genuinely idle, have undrained inbox mail, are not paused /
+ * not awaiting a human decision, and have not been nudged recently — then types
+ * the same guarded nudge the renderer would have, directly into the PTY.
+ *
+ * Safety mirrors the renderer's guarded queue-drain (useHive.ts dispatch):
+ * - only a GENUINELY idle worker is nudged (no PTY output for IDLE_MS — the
+ * same quiescence the renderer's idle fallback uses), never a mid-turn one,
+ * - never inside the boot sequence (BOOT_GRACE_MS from spawn, mirroring the
+ * renderer's bootGraceUntil),
+ * - delivery paused / agent paused / halted → no nudge (ControlRegistry),
+ * - a recent permission/HITL notification re-arms a block (HITL_REARM_MS) so a
+ * prompt the human is deciding on is never typed into,
+ * - a per-worker cooldown (NUDGE_COOLDOWN_MS) so the watchdog and the renderer
+ * nudge don't stack on top of each other.
+ *
+ * Deliberately the renderer's own nudge text, and the same type pattern the
+ * renderer's submitToPty uses (text first, Enter as a separate keystroke).
+ *
+ * No electron import — unit-testable (mirrors ControlRegistry).
+ */
+
+/** The exact nudge the renderer's inbox-wake loop would have typed. */
+export const WORKER_WAKE_NUDGE =
+ 'You have new hive inbox message(s) — read your inbox, act on them now, and move handled ones to inbox/.done/. Act autonomously; only message god if you genuinely need a decision.';
+
+/** No PTY output for this long = genuinely idle (renderer QUIESCE_IDLE_MS). */
+export const WORKER_WAKE_IDLE_MS = 12_000;
+/** Never nudge inside the boot sequence (renderer BOOT_GRACE_MS). */
+export const WORKER_WAKE_BOOT_GRACE_MS = 35_000;
+/** Minimum gap between two watchdog nudges of the same worker. */
+export const WORKER_WAKE_COOLDOWN_MS = 60_000;
+/** A permission/HITL notification blocks nudges for this long after it fires. */
+export const WORKER_WAKE_HITL_REARM_MS = 5 * 60_000;
+
+/** A hook event message that means "the agent needs the human" — permission /
+ * approve / confirm prompts (mirrors the renderer's needsHuman detection in
+ * useHive.ts). Anything matching the idle-waiting shape is NOT a HITL hold. */
+export type HookClass = 'needsHuman' | 'idle' | null;
+
+export function classifyHook(event: string | undefined, message: string | undefined): HookClass {
+ if (event === 'Notification') {
+ const msg = (message ?? '').toLowerCase();
+ const idleWaiting = !msg
+ || msg.includes('waiting for your input')
+ || msg.includes('is idle')
+ || msg.includes('waiting for input');
+ const needsHuman = msg.includes('permission')
+ || msg.includes('approve')
+ || msg.includes('confirm')
+ || msg.includes('needs your');
+ if (needsHuman && !idleWaiting) return 'needsHuman';
+ return 'idle';
+ }
+ return null;
+}
+
+/** One worker's live facts, gathered by the caller each beat. */
+export interface WorkerWakeFacts {
+ /** Worker agent id (god is never a candidate). */
+ agentId: string;
+ /** True when this agent is the orchestrator — god is never nudged. */
+ isGod?: boolean;
+ /** Live PTY id, or undefined when the agent has no terminal. */
+ ptyId?: string;
+ /** Timestamp of the PTY's last output (0 = never output). */
+ lastOutputAt: number;
+ /** Count of undrained inbox messages (0 → nothing to wake for). */
+ inboxCount: number;
+ /** ControlRegistry snapshot flags. */
+ autoDeliveryPaused: boolean;
+ paused: boolean;
+ halted: boolean;
+}
+
+export class WorkerWakeWatchdog {
+ /** ptyId → spawn timestamp (boot grace). */
+ private spawnedAt = new Map<string, number>();
+ /** agentId → last nudge timestamp (cooldown). */
+ private lastNudgeAt = new Map<string, number>();
+ /** agentId → timestamp of the last needsHuman hook notification. */
+ private lastHumanNeedsAt = new Map<string, number>();
+
+ /** Record a PTY spawn so its boot sequence is left alone. */
+ noteSpawn(ptyId: string, at = Date.now()): void {
+ this.spawnedAt.set(ptyId, at);
+ }
+
+ /** Feed hook events (from HookServer) so a HITL prompt blocks nudges. */
+ noteHook(agentId: string | undefined, event: string | undefined, message: string | undefined, at = Date.now()): void {
+ if (!agentId) return;
+ if (classifyHook(event, message) === 'needsHuman') this.lastHumanNeedsAt.set(agentId, at);
+ }
+
+ /** Forget per-agent state (e.g. the agent's PTY was closed). */
+ forget(agentId: string, ptyId?: string): void {
+ this.lastNudgeAt.delete(agentId);
+ this.lastHumanNeedsAt.delete(agentId);
+ if (ptyId) this.spawnedAt.delete(ptyId);
+ }
+
+ /** The worker ids that should be nudged right now, in stable registry order.
+ * Pure decision — the caller types the nudge. */
+ decide(facts: readonly WorkerWakeFacts[], now = Date.now()): string[] {
+ const out: string[] = [];
+ for (const f of facts) {
+ if (f.isGod || f.inboxCount <= 0 || !f.ptyId) continue;
+ if (f.autoDeliveryPaused || f.paused || f.halted) continue;
+ if (f.lastOutputAt <= 0) continue; // never produced output → still booting
+ if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS) continue; // mid-turn
+ const spawned = this.spawnedAt.get(f.ptyId) ?? 0;
+ if (spawned > 0 && now - spawned < WORKER_WAKE_BOOT_GRACE_MS) continue;
+ const lastHuman = this.lastHumanNeedsAt.get(f.agentId) ?? 0;
+ if (lastHuman > 0 && now - lastHuman < WORKER_WAKE_HITL_REARM_MS) continue;
+ const lastNudge = this.lastNudgeAt.get(f.agentId) ?? 0;
+ if (lastNudge > 0 && now - lastNudge < WORKER_WAKE_COOLDOWN_MS) continue;
+ this.lastNudgeAt.set(f.agentId, now);
+ out.push(f.agentId);
+ }
+ return out;
+ }
+
+ /** Last time this worker was nudged (0 = never) — useful for diagnostics. */
+ lastNudge(agentId: string): number {
+ return this.lastNudgeAt.get(agentId) ?? 0;
+ }
+}
\ No newline at end of file
diff --git a/test/worker-wake.test.cjs b/test/worker-wake.test.cjs
new file mode 100644
index 00000000..18478891
--- /dev/null
+++ b/test/worker-wake.test.cjs
@@ -0,0 +1,151 @@
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const loadTs = require('./load-ts.cjs');
+
+const {
+ WorkerWakeWatchdog,
+ classifyHook,
+ WORKER_WAKE_NUDGE,
+ WORKER_WAKE_IDLE_MS,
+ WORKER_WAKE_BOOT_GRACE_MS,
+ WORKER_WAKE_COOLDOWN_MS,
+ WORKER_WAKE_HITL_REARM_MS
+} = loadTs('src/main/workerWake.ts');
+
+/** A permissive fact; tests override the fields they care about. */
+function fact(overrides = {}) {
+ return {
+ agentId: 'alice',
+ ptyId: 'pty-alice',
+ lastOutputAt: 100_000,
+ inboxCount: 1,
+ autoDeliveryPaused: false,
+ paused: false,
+ halted: false,
+ ...overrides
+ };
+}
+
+test('nudges an idle worker with undrained inbox mail', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ const out = w.decide([fact({ lastOutputAt: now - WORKER_WAKE_IDLE_MS - 1 })], now);
+ assert.deepEqual(out, ['alice']);
+});
+
+test('never nudges god, archived agents, or agents without a live pty', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('p1', 0);
+ const now = 200_000;
+ const out = w.decide([
+ fact({ agentId: 'god', isGod: true, ptyId: 'p1' }),
+ fact({ agentId: 'gone', archived: true, ptyId: undefined }),
+ fact({ agentId: 'no-pty', ptyId: undefined })
+ ], now);
+ assert.deepEqual(out, []);
+});
+
+test('never nudges when there is no inbox mail', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ const out = w.decide([fact({ inboxCount: 0 })], now);
+ assert.deepEqual(out, []);
+});
+
+test('never nudges a mid-turn worker (recent PTY output)', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ const out = w.decide([fact({ lastOutputAt: now - 1 })], now);
+ assert.deepEqual(out, []);
+});
+
+test('never nudges a worker that never produced output (still booting)', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ const out = w.decide([fact({ lastOutputAt: 0 })], now);
+ assert.deepEqual(out, []);
+});
+
+test('respects the boot grace window', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 10_000);
+ const now = 10_000 + WORKER_WAKE_BOOT_GRACE_MS - 1;
+ assert.deepEqual(w.decide([fact({ lastOutputAt: 0 })], now), []);
+ // past the grace (and idle long enough) → eligible
+ const late = 10_000 + WORKER_WAKE_BOOT_GRACE_MS + WORKER_WAKE_IDLE_MS + 1;
+ assert.deepEqual(w.decide([fact({ lastOutputAt: late - WORKER_WAKE_IDLE_MS - 1 })], late), ['alice']);
+});
+
+test('never nudges while delivery is paused, agent paused, or halted', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('p1', 0);
+ const now = 200_000;
+ const out = w.decide([
+ fact({ agentId: 'a1', ptyId: 'p1', autoDeliveryPaused: true }),
+ fact({ agentId: 'a2', ptyId: 'p1', paused: true }),
+ fact({ agentId: 'a3', ptyId: 'p1', halted: true })
+ ], now);
+ assert.deepEqual(out, []);
+});
+
+test('a recent permission/HITL notification blocks nudges', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ w.noteHook('alice', 'Notification', 'Claude needs your permission to use Bash.', now - 1_000);
+ assert.deepEqual(w.decide([fact()], now), []);
+ // after the rearm window the block expires
+ const later = now + WORKER_WAKE_HITL_REARM_MS + 1;
+ assert.deepEqual(w.decide([fact({ lastOutputAt: later - WORKER_WAKE_IDLE_MS - 1 })], later), ['alice']);
+});
+
+test('an idle-waiting notification does NOT count as a HITL hold', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ w.noteHook('alice', 'Notification', 'waiting for your input', now - 1_000);
+ assert.deepEqual(w.decide([fact()], now), ['alice']);
+});
+
+test('a nudge is not repeated within the cooldown', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', 0);
+ const now = 200_000;
+ assert.deepEqual(w.decide([fact()], now), ['alice']);
+ assert.deepEqual(w.decide([fact()], now + WORKER_WAKE_COOLDOWN_MS - 1), []);
+ assert.deepEqual(w.decide([fact({ lastOutputAt: now + WORKER_WAKE_COOLDOWN_MS + 1 - WORKER_WAKE_IDLE_MS - 1 })], now + WORKER_WAKE_COOLDOWN_MS + 1), ['alice']);
+});
+
+test('forget clears cooldown + boot grace + HITL state', () => {
+ const w = new WorkerWakeWatchdog();
+ w.noteSpawn('pty-alice', Date.now());
+ w.noteHook('alice', 'Notification', 'permission', Date.now());
+ w.decide([fact()], Date.now());
+ w.forget('alice', 'pty-alice');
+ const now = 200_000;
+ assert.deepEqual(w.decide([fact({ lastOutputAt: now - WORKER_WAKE_IDLE_MS - 1 })], now), ['alice']);
+});
+
+test('classifyHook: permission/approve/confirm shapes are needsHuman', () => {
+ assert.equal(classifyHook('Notification', 'Claude needs your permission to use Bash.'), 'needsHuman');
+ assert.equal(classifyHook('Notification', 'Approve tool use?'), 'needsHuman');
+ assert.equal(classifyHook('Notification', 'confirm the change?'), 'needsHuman');
+});
+
+test('classifyHook: idle-waiting shapes are idle, other events are null', () => {
+ assert.equal(classifyHook('Notification', 'waiting for your input'), 'idle');
+ assert.equal(classifyHook('Notification', 'Claude is idle — waiting for input'), 'idle');
+ assert.equal(classifyHook('Notification', ''), 'idle');
+ assert.equal(classifyHook('Stop', 'some message'), null);
+ assert.equal(classifyHook('UserPromptSubmit', undefined), null);
+});
+
+test('the nudge text matches the renderer guardrail exactly', () => {
+ // If the renderer's nudge wording ever changes, update WORKER_WAKE_NUDGE to match.
+ assert.equal(WORKER_WAKE_NUDGE.length > 100, true);
+ assert.match(WORKER_WAKE_NUDGE, /read your inbox/i);
+});
\ No newline at end of file