an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
fix(worker-wake): main-process watchdog wakes idle workers on undrained inbox
Aravind Rao68cbc25cba67
diff --git a/src/main/hooks.ts b/src/main/hooks.tsindex 52195d84..745d0edf 100644--- a/src/main/hooks.ts+++ b/src/main/hooks.ts@@ -67,7 +67,11 @@ export class HookServer {private control?: ControlRegistry,/** Circuit breaker (Lane A #6.6b) — fed the hook-derived signals (session id,* repeated identical tool calls). Optional so the server still runs without it. */- private breaker?: CircuitBreaker+ private breaker?: CircuitBreaker,+ /** Optional observer of every hook boundary (agentId, event, message). The+ * worker inbox-wake watchdog (workerWake.ts) feeds on this to learn when an+ * agent is parked on a permission/HITL prompt so it never types into it. */+ private onEvent?: (agentId: string | undefined, event: string, message: string | undefined) => void) {}start(): void {@@ -115,6 +119,7 @@ export class HookServer {private handle(p: HookPayload): unknown {const agentId = p.agent_id ?? undefined;const event = p.hook_event_name ?? 'Unknown';+ this.onEvent?.(agentId, event, p.message);if (agentId && typeof p.transcript_path === 'string' && p.transcript_path) {this.transcriptPaths.set(agentId, p.transcript_path);}diff --git a/src/main/index.ts b/src/main/index.tsindex 45d288e4..a2d06b6a 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -59,6 +59,7 @@ import * as integrations from './integrations';import { validateBaseUrl, buildAuthHeaders, resolveUpstreamUrl, secretRefFor, INTEGRATION_TEMPLATES } from '../shared/integrations';import { RosterStore } from './roster';import { ControlRegistry } from './control';+import { WorkerWakeWatchdog, WORKER_WAKE_NUDGE, type WorkerWakeFacts } from './workerWake';import { fetchHireManifest, readHireManifestFile } from './hire';import { parseHireDeepLink, type HireManifest } from '../shared/hire';import { ClosingTimeController } from './closingTime';@@ -262,9 +263,15 @@ let breakerBeatTimer: ReturnType<typeof setInterval> | null = null;// Feed the breaker's api_error-storm trip from Oscar's OTel api_error spans —// Jim's one breaker input with no on-branch source (telemetry.onApiError seam).telemetry.onApiError((agentId) => breaker.recordError(agentId));+// Worker inbox-wake watchdog (#151): finds idle workers with undrained inbox mail+// and types the same guarded nudge the renderer would have (so a throttled+// background window can't leave a worker parked on an unread inbox forever).+// HookServer feeds it the hook stream so a permission/HITL prompt blocks nudges.+const workerWake = new WorkerWakeWatchdog();// HookServer needs BOTH: Oscar's control registry (HITL pause/gate/steer/halt via// hook returns) AND Jim's breaker (feed recordToolUse on each PostToolUse).-const hookServer = new HookServer(hive, () => liveWebContents(), () => readConfig(), control, breaker);+const hookServer = new HookServer(hive, () => liveWebContents(), () => readConfig(), control, breaker,+ (agentId, event, message) => workerWake.noteHook(agentId, event, message));const memory = new MemoryManager(() => readConfig().harnessHome,() => { const c = readConfig(); return { enabled: c.semanticMemory !== false, model: c.embeddingModel ?? 'minilm' }; }@@ -398,6 +405,8 @@ function teardownPty(id: string): void {const agentId = ptyToAgent.get(id);if (agentId) {ptyToAgent.delete(id);+ // Drop watchdog state so a dead agent can't get nudged or leak its grace.+ try { workerWake.forget(agentId, id); } catch { /* best-effort */ }// Drop breaker state so a dead agent can't leak/zombie a tripped level.try { breaker.forget(agentId); } catch { /* best-effort */ }// W1 — kill this agent's proxy-bridge sidecar (qwen), if any, so a dead@@ -2681,7 +2690,12 @@ async function spawnAgentCore(opts: AgentSpawnOptions, owner: Electron.WebConten}// Remember which agent owns this PTY so closing the tab can archive it. A// live terminal means active — ensureAgent above already cleared `archived`.- if (opts.hive?.id) ptyToAgent.set(opts.id, opts.hive.id);+ if (opts.hive?.id) {+ ptyToAgent.set(opts.id, opts.hive.id);+ // Worker inbox-wake watchdog (#151): boot grace starts at spawn so the+ // initial orientation prompt is never mistaken for an idle agent.+ workerWake.noteSpawn(opts.id);+ }// Pre-accept Claude Code's bypass-mode warning + folder-trust dialog so the// agent (spawned with --permission-mode bypassPermissions) doesn't stall on an// interactive prompt it can't answer and exit code 1. Best-effort, never blocks.@@ -4623,6 +4637,64 @@ function bootstrapHiveServices(): void {armAlwaysOnBeats();}+/** Cadence of the worker inbox-wake watchdog (#151). Well under the renderer's+ * own nudge cooldown so a throttled window is caught within ~15s of a stall. */+const WORKER_WAKE_POLL_MS = 15_000;+let workerWakeTimer: ReturnType<typeof setInterval> | null = null;++/** Type the renderer's guarded nudge into one worker's PTY — text first, Enter a+ * tick later (the exact submitToPty pattern: a single-chunk write would land the+ * "\r" inside the input box and never submit). Best-effort + never throws. */+function nudgeWorker(ptyId: string): void {+ const wrote = ptyManager.write(ptyId, WORKER_WAKE_NUDGE);+ if (!wrote.ok) { console.warn(`[worker-wake] write failed for ${ptyId}: ${wrote.error}`); return; }+ setTimeout(() => {+ try {+ const submitted = ptyManager.write(ptyId, '\r');+ if (!submitted.ok) console.warn(`[worker-wake] submit failed for ${ptyId}: ${submitted.error}`);+ } catch (e) { console.error('[worker-wake] submit threw:', e); }+ }, 140);+}++/** Main-process inbox-wake beat (issue #151, fix A): the renderer's idle nudge+ * (useHive.ts) is the only path that wakes a worker parked on an undrained+ * inbox — and it lives on a setInterval in the renderer, which a throttled or+ * occluded window stops honoring. This beat is the renderer-INDEPENDENT fallback:+ * it gathers live-worker facts (PTY quiescence, inbox depth, control flags) and+ * lets WorkerWakeWatchdog.decide apply the exact renderer guards (idle-only,+ * post-boot-grace, not paused/halted, no pending HITL, cooldown), then types the+ * same nudge the renderer would have. God is never a candidate (its heartbeat+ * path already re-engages it). */+function runWorkerWakeBeat(): void {+ if (!hive.enabled()) return;+ const reg = hive.registry();+ if (!reg?.agents || !reg.godId) return;+ const now = Date.now();+ const facts: WorkerWakeFacts[] = [];+ for (const [agentId, a] of Object.entries(reg.agents)) {+ if (agentId === reg.godId || a?.archived) continue;+ const ptyId = ptyForAgent(agentId);+ if (!ptyId) continue;+ const snap = control.snapshot(agentId);+ facts.push({+ agentId,+ isGod: agentId === reg.godId,+ ptyId,+ lastOutputAt: ptyManager.lastOutputAt(ptyId) ?? 0,+ inboxCount: hive.inbox(agentId).length,+ autoDeliveryPaused: snap.autoDeliveryPaused,+ paused: snap.paused,+ halted: snap.halted+ });+ }+ for (const agentId of workerWake.decide(facts, now)) {+ const ptyId = ptyForAgent(agentId);+ if (!ptyId) continue;+ console.log(`[worker-wake] nudging ${agentId} on ${ptyId}`);+ nudgeWorker(ptyId);+ }+}+/** (Re)arm the always-on beats (decoupled from the optional heartbeat): the live* fleet snapshot Michael reads (~8s) + the breaker/cost-ledger beat (~30s).* Guarded (clear-then-set) so a re-bootstrap (changeHome recovery) OR a@@ -4634,6 +4706,9 @@ function armAlwaysOnBeats(): void {fleetTimer = setInterval(writeFleetSnapshot, 8_000);if (breakerBeatTimer) clearInterval(breakerBeatTimer);breakerBeatTimer = setInterval(() => { try { runBreakerBeat(300_000); } catch (e) { console.error('[breaker beat]', e); } }, 30_000);+ if (workerWakeTimer) clearInterval(workerWakeTimer);+ workerWakeTimer = setInterval(() => { try { runWorkerWakeBeat(); } catch (e) { console.error('[worker-wake beat]', e); } }, WORKER_WAKE_POLL_MS);+ runWorkerWakeBeat(); // catch-up on arm — power-resume re-arms and drains the backlog}/** Wall-clock instant we last observed the machine suspend or lock, so a resumediff --git a/src/main/workerWake.ts b/src/main/workerWake.tsnew file mode 100644index 00000000..feadcea2--- /dev/null+++ b/src/main/workerWake.ts@@ -0,0 +1,137 @@+/**+ * WorkerWakeWatchdog — main-process inbox-wake watchdog for worker agents (#151).+ *+ * The renderer's idle inbox-wake nudge (useHive.ts effect #3) is the ONLY wake+ * path for a worker that has gone quiet at its prompt: it polls on a setInterval+ * in the renderer, so a throttled/occluded window (Chromium suspends background+ * setInterval timers) can miss the moment mail lands and the worker then sits on+ * an undrained inbox forever — the orchestrator ("god") never has this problem+ * because the main process re-engages it on its own heartbeat cadence.+ *+ * This watchdog is the worker-side counterpart: on a cadence it finds live+ * workers that are genuinely idle, have undrained inbox mail, are not paused /+ * not awaiting a human decision, and have not been nudged recently — then types+ * the same guarded nudge the renderer would have, directly into the PTY.+ *+ * Safety mirrors the renderer's guarded queue-drain (useHive.ts dispatch):+ * - only a GENUINELY idle worker is nudged (no PTY output for IDLE_MS — the+ * same quiescence the renderer's idle fallback uses), never a mid-turn one,+ * - never inside the boot sequence (BOOT_GRACE_MS from spawn, mirroring the+ * renderer's bootGraceUntil),+ * - delivery paused / agent paused / halted → no nudge (ControlRegistry),+ * - a recent permission/HITL notification re-arms a block (HITL_REARM_MS) so a+ * prompt the human is deciding on is never typed into,+ * - a per-worker cooldown (NUDGE_COOLDOWN_MS) so the watchdog and the renderer+ * nudge don't stack on top of each other.+ *+ * Deliberately the renderer's own nudge text, and the same type pattern the+ * renderer's submitToPty uses (text first, Enter as a separate keystroke).+ *+ * No electron import — unit-testable (mirrors ControlRegistry).+ */++/** The exact nudge the renderer's inbox-wake loop would have typed. */+export const WORKER_WAKE_NUDGE =+ 'You have new hive inbox message(s) — read your inbox, act on them now, and move handled ones to inbox/.done/. Act autonomously; only message god if you genuinely need a decision.';++/** No PTY output for this long = genuinely idle (renderer QUIESCE_IDLE_MS). */+export const WORKER_WAKE_IDLE_MS = 12_000;+/** Never nudge inside the boot sequence (renderer BOOT_GRACE_MS). */+export const WORKER_WAKE_BOOT_GRACE_MS = 35_000;+/** Minimum gap between two watchdog nudges of the same worker. */+export const WORKER_WAKE_COOLDOWN_MS = 60_000;+/** A permission/HITL notification blocks nudges for this long after it fires. */+export const WORKER_WAKE_HITL_REARM_MS = 5 * 60_000;++/** A hook event message that means "the agent needs the human" — permission /+ * approve / confirm prompts (mirrors the renderer's needsHuman detection in+ * useHive.ts). Anything matching the idle-waiting shape is NOT a HITL hold. */+export type HookClass = 'needsHuman' | 'idle' | null;++export function classifyHook(event: string | undefined, message: string | undefined): HookClass {+ if (event === 'Notification') {+ const msg = (message ?? '').toLowerCase();+ const idleWaiting = !msg+ || msg.includes('waiting for your input')+ || msg.includes('is idle')+ || msg.includes('waiting for input');+ const needsHuman = msg.includes('permission')+ || msg.includes('approve')+ || msg.includes('confirm')+ || msg.includes('needs your');+ if (needsHuman && !idleWaiting) return 'needsHuman';+ return 'idle';+ }+ return null;+}++/** One worker's live facts, gathered by the caller each beat. */+export interface WorkerWakeFacts {+ /** Worker agent id (god is never a candidate). */+ agentId: string;+ /** True when this agent is the orchestrator — god is never nudged. */+ isGod?: boolean;+ /** Live PTY id, or undefined when the agent has no terminal. */+ ptyId?: string;+ /** Timestamp of the PTY's last output (0 = never output). */+ lastOutputAt: number;+ /** Count of undrained inbox messages (0 → nothing to wake for). */+ inboxCount: number;+ /** ControlRegistry snapshot flags. */+ autoDeliveryPaused: boolean;+ paused: boolean;+ halted: boolean;+}++export class WorkerWakeWatchdog {+ /** ptyId → spawn timestamp (boot grace). */+ private spawnedAt = new Map<string, number>();+ /** agentId → last nudge timestamp (cooldown). */+ private lastNudgeAt = new Map<string, number>();+ /** agentId → timestamp of the last needsHuman hook notification. */+ private lastHumanNeedsAt = new Map<string, number>();++ /** Record a PTY spawn so its boot sequence is left alone. */+ noteSpawn(ptyId: string, at = Date.now()): void {+ this.spawnedAt.set(ptyId, at);+ }++ /** Feed hook events (from HookServer) so a HITL prompt blocks nudges. */+ noteHook(agentId: string | undefined, event: string | undefined, message: string | undefined, at = Date.now()): void {+ if (!agentId) return;+ if (classifyHook(event, message) === 'needsHuman') this.lastHumanNeedsAt.set(agentId, at);+ }++ /** Forget per-agent state (e.g. the agent's PTY was closed). */+ forget(agentId: string, ptyId?: string): void {+ this.lastNudgeAt.delete(agentId);+ this.lastHumanNeedsAt.delete(agentId);+ if (ptyId) this.spawnedAt.delete(ptyId);+ }++ /** The worker ids that should be nudged right now, in stable registry order.+ * Pure decision — the caller types the nudge. */+ decide(facts: readonly WorkerWakeFacts[], now = Date.now()): string[] {+ const out: string[] = [];+ for (const f of facts) {+ if (f.isGod || f.inboxCount <= 0 || !f.ptyId) continue;+ if (f.autoDeliveryPaused || f.paused || f.halted) continue;+ if (f.lastOutputAt <= 0) continue; // never produced output → still booting+ if (now - f.lastOutputAt < WORKER_WAKE_IDLE_MS) continue; // mid-turn+ const spawned = this.spawnedAt.get(f.ptyId) ?? 0;+ if (spawned > 0 && now - spawned < WORKER_WAKE_BOOT_GRACE_MS) continue;+ const lastHuman = this.lastHumanNeedsAt.get(f.agentId) ?? 0;+ if (lastHuman > 0 && now - lastHuman < WORKER_WAKE_HITL_REARM_MS) continue;+ const lastNudge = this.lastNudgeAt.get(f.agentId) ?? 0;+ if (lastNudge > 0 && now - lastNudge < WORKER_WAKE_COOLDOWN_MS) continue;+ this.lastNudgeAt.set(f.agentId, now);+ out.push(f.agentId);+ }+ return out;+ }++ /** Last time this worker was nudged (0 = never) — useful for diagnostics. */+ lastNudge(agentId: string): number {+ return this.lastNudgeAt.get(agentId) ?? 0;+ }+}\ No newline at end of filediff --git a/test/worker-wake.test.cjs b/test/worker-wake.test.cjsnew file mode 100644index 00000000..18478891--- /dev/null+++ b/test/worker-wake.test.cjs@@ -0,0 +1,151 @@+const { test } = require('node:test');+const assert = require('node:assert/strict');+const loadTs = require('./load-ts.cjs');++const {+ WorkerWakeWatchdog,+ classifyHook,+ WORKER_WAKE_NUDGE,+ WORKER_WAKE_IDLE_MS,+ WORKER_WAKE_BOOT_GRACE_MS,+ WORKER_WAKE_COOLDOWN_MS,+ WORKER_WAKE_HITL_REARM_MS+} = loadTs('src/main/workerWake.ts');++/** A permissive fact; tests override the fields they care about. */+function fact(overrides = {}) {+ return {+ agentId: 'alice',+ ptyId: 'pty-alice',+ lastOutputAt: 100_000,+ inboxCount: 1,+ autoDeliveryPaused: false,+ paused: false,+ halted: false,+ ...overrides+ };+}++test('nudges an idle worker with undrained inbox mail', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ const out = w.decide([fact({ lastOutputAt: now - WORKER_WAKE_IDLE_MS - 1 })], now);+ assert.deepEqual(out, ['alice']);+});++test('never nudges god, archived agents, or agents without a live pty', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('p1', 0);+ const now = 200_000;+ const out = w.decide([+ fact({ agentId: 'god', isGod: true, ptyId: 'p1' }),+ fact({ agentId: 'gone', archived: true, ptyId: undefined }),+ fact({ agentId: 'no-pty', ptyId: undefined })+ ], now);+ assert.deepEqual(out, []);+});++test('never nudges when there is no inbox mail', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ const out = w.decide([fact({ inboxCount: 0 })], now);+ assert.deepEqual(out, []);+});++test('never nudges a mid-turn worker (recent PTY output)', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ const out = w.decide([fact({ lastOutputAt: now - 1 })], now);+ assert.deepEqual(out, []);+});++test('never nudges a worker that never produced output (still booting)', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ const out = w.decide([fact({ lastOutputAt: 0 })], now);+ assert.deepEqual(out, []);+});++test('respects the boot grace window', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 10_000);+ const now = 10_000 + WORKER_WAKE_BOOT_GRACE_MS - 1;+ assert.deepEqual(w.decide([fact({ lastOutputAt: 0 })], now), []);+ // past the grace (and idle long enough) → eligible+ const late = 10_000 + WORKER_WAKE_BOOT_GRACE_MS + WORKER_WAKE_IDLE_MS + 1;+ assert.deepEqual(w.decide([fact({ lastOutputAt: late - WORKER_WAKE_IDLE_MS - 1 })], late), ['alice']);+});++test('never nudges while delivery is paused, agent paused, or halted', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('p1', 0);+ const now = 200_000;+ const out = w.decide([+ fact({ agentId: 'a1', ptyId: 'p1', autoDeliveryPaused: true }),+ fact({ agentId: 'a2', ptyId: 'p1', paused: true }),+ fact({ agentId: 'a3', ptyId: 'p1', halted: true })+ ], now);+ assert.deepEqual(out, []);+});++test('a recent permission/HITL notification blocks nudges', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ w.noteHook('alice', 'Notification', 'Claude needs your permission to use Bash.', now - 1_000);+ assert.deepEqual(w.decide([fact()], now), []);+ // after the rearm window the block expires+ const later = now + WORKER_WAKE_HITL_REARM_MS + 1;+ assert.deepEqual(w.decide([fact({ lastOutputAt: later - WORKER_WAKE_IDLE_MS - 1 })], later), ['alice']);+});++test('an idle-waiting notification does NOT count as a HITL hold', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ w.noteHook('alice', 'Notification', 'waiting for your input', now - 1_000);+ assert.deepEqual(w.decide([fact()], now), ['alice']);+});++test('a nudge is not repeated within the cooldown', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', 0);+ const now = 200_000;+ assert.deepEqual(w.decide([fact()], now), ['alice']);+ assert.deepEqual(w.decide([fact()], now + WORKER_WAKE_COOLDOWN_MS - 1), []);+ assert.deepEqual(w.decide([fact({ lastOutputAt: now + WORKER_WAKE_COOLDOWN_MS + 1 - WORKER_WAKE_IDLE_MS - 1 })], now + WORKER_WAKE_COOLDOWN_MS + 1), ['alice']);+});++test('forget clears cooldown + boot grace + HITL state', () => {+ const w = new WorkerWakeWatchdog();+ w.noteSpawn('pty-alice', Date.now());+ w.noteHook('alice', 'Notification', 'permission', Date.now());+ w.decide([fact()], Date.now());+ w.forget('alice', 'pty-alice');+ const now = 200_000;+ assert.deepEqual(w.decide([fact({ lastOutputAt: now - WORKER_WAKE_IDLE_MS - 1 })], now), ['alice']);+});++test('classifyHook: permission/approve/confirm shapes are needsHuman', () => {+ assert.equal(classifyHook('Notification', 'Claude needs your permission to use Bash.'), 'needsHuman');+ assert.equal(classifyHook('Notification', 'Approve tool use?'), 'needsHuman');+ assert.equal(classifyHook('Notification', 'confirm the change?'), 'needsHuman');+});++test('classifyHook: idle-waiting shapes are idle, other events are null', () => {+ assert.equal(classifyHook('Notification', 'waiting for your input'), 'idle');+ assert.equal(classifyHook('Notification', 'Claude is idle — waiting for input'), 'idle');+ assert.equal(classifyHook('Notification', ''), 'idle');+ assert.equal(classifyHook('Stop', 'some message'), null);+ assert.equal(classifyHook('UserPromptSubmit', undefined), null);+});++test('the nudge text matches the renderer guardrail exactly', () => {+ // If the renderer's nudge wording ever changes, update WORKER_WAKE_NUDGE to match.+ assert.equal(WORKER_WAKE_NUDGE.length > 100, true);+ assert.match(WORKER_WAKE_NUDGE, /read your inbox/i);+});\ No newline at end of file