Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

fix(hive): main-process inbox-wake watchdog for workers (#151)

Gastón Péchieu8c0453447043
diff --git a/package.json b/package.json
index 8dc851de..1009eb14 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
"typecheck:node": "tsc --noEmit -p tsconfig.node.json",
"typecheck:web": "tsc --noEmit -p tsconfig.web.json",
"typecheck": "npm run typecheck:node && npm run typecheck:web",
- "test:focused": "node --test test/codex-remote.test.cjs test/control.test.cjs test/provider-automation.test.cjs test/provider-config.test.cjs test/queue-delivery.test.cjs test/terminal-automation.test.cjs test/terminal-recovery.test.cjs test/roster.test.cjs test/proc-kill.test.cjs test/expand-tilde.test.cjs test/hive-cwd.test.cjs test/hive-hook-node.test.cjs test/hive-roster-injection.test.cjs test/hive-runtime-path.test.cjs test/cli-install-ladder.test.cjs test/node-install.test.cjs test/office-gl-recovery.test.cjs test/update-state.test.cjs test/transcript-project-dir.test.cjs test/commit-graph.test.cjs",
+ "test:focused": "node --test test/codex-remote.test.cjs test/control.test.cjs test/provider-automation.test.cjs test/provider-config.test.cjs test/queue-delivery.test.cjs test/terminal-automation.test.cjs test/terminal-recovery.test.cjs test/roster.test.cjs test/proc-kill.test.cjs test/expand-tilde.test.cjs test/hive-cwd.test.cjs test/hive-hook-node.test.cjs test/hive-roster-injection.test.cjs test/hive-runtime-path.test.cjs test/cli-install-ladder.test.cjs test/node-install.test.cjs test/office-gl-recovery.test.cjs test/update-state.test.cjs test/transcript-project-dir.test.cjs test/commit-graph.test.cjs test/inbox-wake.test.cjs",
"check:links": "node tools/check-release-links.cjs",
"dist": "npm run build && electron-builder",
"dist:mac": "npm run build && electron-builder --mac",
diff --git a/src/main/hooks.ts b/src/main/hooks.ts
index 52195d84..a519d194 100644
--- a/src/main/hooks.ts
+++ b/src/main/hooks.ts
@@ -58,6 +58,8 @@ export class HookServer {
* get_agent_detail / list_agents) can report "how full is each agent's context"
* without depending on a renderer round-trip. */
private contextById = new Map<string, { tokens: number; limit: number; ts: number }>();
+ /** #151: per-agent turn state derived from real hook boundaries (see hookIdleFor). */
+ private activity = new Map<string, { idle: boolean; since: number }>();
constructor(
private hive: HiveManager,
@@ -112,6 +114,18 @@ export class HookServer {
return this.contextById.get(agentId);
}
+ /** ms the agent has been hook-idle — 0 while a turn is in flight (or a
+ * notification/HITL prompt is pending) — or null when this agent has emitted
+ * no hook events at all (hookless provider, or not yet booted). The
+ * inbox-wake watchdog (#151) prefers this over raw pty quiet: claude's TUI
+ * repaints its idle prompt constantly, so pty output recency cannot tell
+ * "mid-turn" from "sitting at the prompt". */
+ hookIdleFor(agentId: string): number | null {
+ const a = this.activity.get(agentId);
+ if (!a) return null;
+ return a.idle ? Date.now() - a.since : 0;
+ }
+
private handle(p: HookPayload): unknown {
const agentId = p.agent_id ?? undefined;
const event = p.hook_event_name ?? 'Unknown';
@@ -149,6 +163,20 @@ export class HookServer {
return {};
}
+ // #151: authoritative idle signal for the inbox-wake watchdog. Stop = a
+ // turn just ended; SessionStart = the CLI is sitting at its prompt (fresh
+ // spawn or --resume — without it a restarted agent that never ran a turn
+ // would read busy forever). Any other real boundary (tool use, prompt
+ // submit, a notification/HITL prompt, compaction, a subagent stopping
+ // while its parent runs) = busy. Status ticks returned above never reach
+ // here, so pure telemetry can't fake activity.
+ if (agentId) {
+ this.activity.set(agentId, {
+ idle: event === 'Stop' || event === 'SessionStart',
+ since: Date.now()
+ });
+ }
+
// 7C.3 — a graceful operator HALT overrides everything (incl. the inbox
// drain below): stop the agent CLEANLY at this hook boundary rather than
// killing the PTY. session_id is in the payload for a later --resume.
diff --git a/src/main/inboxWake.ts b/src/main/inboxWake.ts
new file mode 100644
index 00000000..b6aaf714
--- /dev/null
+++ b/src/main/inboxWake.ts
@@ -0,0 +1,81 @@
+/**
+ * Worker inbox-wake watchdog — the main-process half of issue #151.
+ *
+ * Worker wake used to live ONLY in the renderer: one 4s interval enqueues an
+ * inbox nudge (deduped on the newest message id), another drains it into the
+ * terminal once the agent idles. One wedged store status, one stale dedup key,
+ * or one renderer reload losing its in-memory queue, and a durable inbox file
+ * sat unread until a full app restart. God never had this problem: the
+ * heartbeat re-delivers HIS digest from main on a cadence. This gives workers
+ * the same second engine.
+ *
+ * DECISION ONLY — no I/O, no timers. The index.ts beat feeds it the live
+ * registry + pty idle times and broadcasts one `hive:inboxWake` per firing; the
+ * RENDERER stays the only terminal writer, re-driving its normal guarded
+ * delivery (idle-only, boot grace, pause, draft/picker safety), so this can
+ * never bypass the HITL protections hooks.ts deliberately preserves at Stop
+ * (never force a continuation from unread mail).
+ */
+
+/** An agent must be THIS long idle before it counts as stalled-with-mail —
+ * long enough that a turn wrapping up doesn't get a redundant wake. Idle is
+ * measured from hook boundaries when the provider has them (Stop = idle; any
+ * other real event = busy): raw pty quiet CANNOT tell "mid-turn" from
+ * "sitting at the prompt" for claude's TUI, whose idle prompt repaints
+ * constantly. Hookless providers fall back to pty output recency. */
+export const WAKE_QUIET_MS = 30_000;
+/** While the same mail stays undrained, re-fire on this cadence — the retry
+ * the renderer's single-shot nudge never had. */
+export const WAKE_RETRY_MS = 5 * 60_000;
+
+export interface InboxWakeDeps {
+ /** Live registry entries (archived included — they are skipped here). */
+ agents: Record<string, { archived?: boolean }>;
+ godId: string | null | undefined;
+ /** Live pty id for an agent, or undefined when it has none. */
+ ptyFor(agentId: string): string | undefined;
+ /** ms the agent has been hook-idle (0 = mid-turn / HITL prompt pending), or
+ * null when it has emitted no hook events (hookless provider / booting). */
+ hookIdleFor(agentId: string): number | null;
+ /** Fallback for hookless agents: ms since the pty last produced output. */
+ idleFor(ptyId: string): number | null | undefined;
+ /** Ids of the messages currently in the agent's inbox. */
+ inboxIds(agentId: string): string[];
+}
+
+export interface InboxWakeFire {
+ agentId: string;
+ /** Same key the renderer's nudge dedup uses (lexically-largest inbox id). */
+ newestId: string;
+ idleMs: number;
+ count: number;
+}
+
+/** Per-agent memory of the last firing — owned by the caller so a re-bootstrap
+ * (changeHome) or a test can reset it. */
+export type InboxWakeState = Map<string, { newestId: string; at: number }>;
+
+export function inboxWakeTick(
+ deps: InboxWakeDeps,
+ state: InboxWakeState,
+ now: number
+): InboxWakeFire[] {
+ const fires: InboxWakeFire[] = [];
+ for (const [id, a] of Object.entries(deps.agents)) {
+ if (a.archived || id === deps.godId) continue; // god has the heartbeat
+ const ptyId = deps.ptyFor(id);
+ if (!ptyId) continue; // no live terminal to wake
+ const ids = deps.inboxIds(id);
+ if (ids.length === 0) { state.delete(id); continue; } // drained — forget it
+ const idleMs = deps.hookIdleFor(id) ?? deps.idleFor(ptyId) ?? 0;
+ if (idleMs < WAKE_QUIET_MS) continue; // mid-turn; it reads mail on its own
+ const newestId = [...ids].sort().slice(-1)[0] ?? '';
+ const prev = state.get(id);
+ // New mail re-fires immediately; the SAME undrained mail re-fires only on
+ // the retry cadence (the renderer dedupes by queue content either way).
+ if (prev && prev.newestId === newestId && now - prev.at < WAKE_RETRY_MS) continue;
+ state.set(id, { newestId, at: now });
+ fires.push({ agentId: id, newestId, idleMs, count: ids.length });
+ }
+ return fires;
+}
diff --git a/src/main/index.ts b/src/main/index.ts
index cb30da18..09c1f245 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -23,6 +23,7 @@ import {
getLogGraph, getCommitFiles, getFileAtRev, compareRefs, listWorktrees, checkoutRef
} from './git';
import { HiveManager, type AgentMeta, type HiveMessage, type HiveTask } from './hive';
+import { inboxWakeTick, type InboxWakeState } from './inboxWake';
import { HookServer } from './hooks';
import { CircuitBreaker, type BreakerInput } from './breaker';
import type { UsageProvider } from './usage';
@@ -255,6 +256,7 @@ const breaker = new CircuitBreaker(() => {
// heartbeat mission is disabled (it ships off).
let fleetTimer: ReturnType<typeof setInterval> | null = null;
let breakerBeatTimer: ReturnType<typeof setInterval> | null = null;
+let inboxWakeTimer: ReturnType<typeof setInterval> | null = null;
// Feed the breaker's api_error-storm trip from Oscar's OTel api_error spans —
// Jim's one breaker input with no on-branch source (telemetry.onApiError seam).
telemetry.onApiError((agentId) => breaker.recordError(agentId));
@@ -1092,6 +1094,36 @@ function runBreakerBeat(progressWindowMs: number): void {
}
}
+/** Workers' second wake engine (#151, decision logic in src/main/inboxWake.ts).
+ * Detection runs HERE because main's timers survive everything the renderer's
+ * don't (reloads, wedged store state, a lost in-memory queue); delivery stays
+ * in the renderer so every existing typing guard keeps holding. Mirrors what
+ * reengageGod already does for the orchestrator. */
+const inboxWakeState: InboxWakeState = new Map();
+function runInboxWakeBeat(): void {
+ if (!hive.enabled()) return;
+ const reg = hive.registry();
+ const fires = inboxWakeTick(
+ {
+ agents: reg.agents,
+ godId: reg.godId,
+ ptyFor: ptyForAgent,
+ hookIdleFor: (id) => hookServer.hookIdleFor(id),
+ idleFor: (ptyId) => ptyManager.idleFor(ptyId),
+ inboxIds: (id) => hive.inbox(id).map((m) => m.id)
+ },
+ inboxWakeState,
+ Date.now()
+ );
+ for (const f of fires) {
+ console.log(
+ `[inbox-wake] ${f.agentId}: ${f.count} undrained message(s), pty quiet ` +
+ `${Math.round(f.idleMs / 1000)}s — re-firing wake`
+ );
+ try { liveWebContents()?.send('hive:inboxWake', f); } catch { /* window torn down */ }
+ }
+}
+
/** Build + write the live fleet snapshot Michael reads (`<hive>/fleet.json`).
* Always-on (independent of the heartbeat) since `claude agents` can't see the
* hive's sibling sessions. PII-free; never throws (called from a timer). */
@@ -4425,7 +4457,8 @@ function bootstrapHiveServices(): void {
}
/** (Re)arm the always-on beats (decoupled from the optional heartbeat): the live
- * fleet snapshot Michael reads (~8s) + the breaker/cost-ledger beat (~30s).
+ * fleet snapshot Michael reads (~8s) + the breaker/cost-ledger beat (~30s) +
+ * the worker inbox-wake watchdog (~60s, #151).
* Guarded (clear-then-set) so a re-bootstrap (changeHome recovery) OR a
* powerMonitor resume can't stack duplicate timers — these are setInterval
* handles that freeze during true system sleep and must be re-armed on wake. */
@@ -4435,6 +4468,8 @@ function armAlwaysOnBeats(): void {
fleetTimer = setInterval(writeFleetSnapshot, 8_000);
if (breakerBeatTimer) clearInterval(breakerBeatTimer);
breakerBeatTimer = setInterval(() => { try { runBreakerBeat(300_000); } catch (e) { console.error('[breaker beat]', e); } }, 30_000);
+ if (inboxWakeTimer) clearInterval(inboxWakeTimer);
+ inboxWakeTimer = setInterval(() => { try { runInboxWakeBeat(); } catch (e) { console.error('[inbox-wake]', e); } }, 60_000);
}
/** Wall-clock instant we last observed the machine suspend or lock, so a resume
diff --git a/src/preload/index.ts b/src/preload/index.ts
index a04efc67..593d9d30 100644
--- a/src/preload/index.ts
+++ b/src/preload/index.ts
@@ -815,6 +815,16 @@ const api = {
ipcRenderer.on('hive:agentArchived', listener);
return () => ipcRenderer.removeListener('hive:agentArchived', listener);
},
+ /** The main-process inbox-wake watchdog (#151): fired when an agent's pty has
+ * sat quiet while inbox mail stays undrained. The renderer re-drives its
+ * normal guarded nudge/delivery — it remains the only terminal writer. */
+ onHiveInboxWake: (
+ cb: (e: { agentId: string; newestId: string; idleMs: number; count: number }) => void
+ ): (() => void) => {
+ const listener = (_e: IpcRendererEvent, payload: Parameters<typeof cb>[0]) => cb(payload);
+ ipcRenderer.on('hive:inboxWake', listener);
+ return () => ipcRenderer.removeListener('hive:inboxWake', listener);
+ },
/** Register a listener for terminal work-order handoffs (#53) — hive mail to a
* hookless provider that can't drain an inbox; the renderer types it into the
* agent's REPL as a work order. */
diff --git a/src/renderer/src/hooks/useHive.ts b/src/renderer/src/hooks/useHive.ts
index c7b10fd3..62063381 100644
--- a/src/renderer/src/hooks/useHive.ts
+++ b/src/renderer/src/hooks/useHive.ts
@@ -46,6 +46,12 @@ const BOOT_GRACE_MS = 35_000;
// submitToPty additionally waits for the terminal's readiness handshake.
const SEED_BOOT_MS = 12_000;
+// The standard inbox-wake nudge — ONE shared string because two paths queue it
+// (effect #3's poll and the main watchdog's re-fire, #151) and dedupe against
+// the queue by content so an agent is never nudged twice for the same mail.
+const INBOX_NUDGE =
+ 'You have new hive inbox message(s) — read your inbox, act on them now, and move handled ones to inbox/.done/. Act autonomously; only message god if you genuinely need a decision.';
+
// The first thing Michael (god) is told on a fresh spawn — orient him and put
// him to work running the floor. Kept terse and action-oriented.
const INITIAL_GOD_PROMPT = [
@@ -609,10 +615,7 @@ export function useHive(config: HarnessConfig | null): void {
? inbox.map((m) => m.id).sort().slice(-1)[0]
: '';
if (newest && nudged.current[a.id] !== newest) {
- useStore.getState().enqueueMessage(
- a.id,
- 'You have new hive inbox message(s) — read your inbox, act on them now, and move handled ones to inbox/.done/. Act autonomously; only message god if you genuinely need a decision.'
- );
+ useStore.getState().enqueueMessage(a.id, INBOX_NUDGE);
nudged.current[a.id] = newest;
} else if (!newest) {
nudged.current[a.id] = '';
@@ -623,6 +626,36 @@ export function useHive(config: HarnessConfig | null): void {
return () => clearInterval(iv);
}, [config?.onboardingComplete]);
+ // 3c) The main-process wake watchdog (#151). Effect #3 is a single renderer
+ // timer with a single-shot dedup — one wedged store status or one stale
+ // key and a durable inbox message stalls until app restart (god survives
+ // this because the heartbeat re-delivers HIS digest from main). Main
+ // re-fires when a pty sits quiet with mail undrained; we re-drive the
+ // SAME queue + drain path, so every typing guard (idle-only, boot grace,
+ // pause, draft/picker safety) still holds.
+ useEffect(() => {
+ if (!config?.onboardingComplete) return;
+ return window.cth.onHiveInboxWake?.((p) => {
+ if (!p?.agentId) return;
+ const { agents, messageQueues, enqueueMessage, updateAgent } = useStore.getState();
+ const a = agents.find((x) => x.id === p.agentId);
+ if (!a?.ptyId) return;
+ // Main has watched this pty stay quiet ≥30s while the store still says
+ // 'working' — that status is wedged (a live turn repaints constantly) and
+ // it is exactly what blocks the drain. Reconcile to idle the same way the
+ // quiesce poll (#2) does. 'waiting'/'blocked'/'looping' are deliberate
+ // holds (HITL prompt / breaker) and are never overridden.
+ if (a.status === 'working' && (p.idleMs ?? 0) >= 30_000) {
+ updateAgent(a.id, { status: 'idle', action: 'awaiting', carrying: undefined });
+ }
+ // Queue the standard nudge unless one is already waiting — and remember
+ // the newest id so effect #3's own tick doesn't queue a duplicate.
+ const queued = messageQueues[a.id] ?? [];
+ if (!queued.some((m) => m.text === INBOX_NUDGE)) enqueueMessage(a.id, INBOX_NUDGE);
+ if (p.newestId) nudged.current[a.id] = p.newestId;
+ });
+ }, [config?.onboardingComplete]);
+
// 3b) Seed a fresh "type-into-tui" worker (Crush) with the hive protocol. Its
// bare TUI rejects a positional seed (Cobra reads it as a subcommand →
// `Unknown command`), so the main process spawns it bare and hands the
diff --git a/test/inbox-wake.test.cjs b/test/inbox-wake.test.cjs
new file mode 100644
index 00000000..4f8a4dc7
--- /dev/null
+++ b/test/inbox-wake.test.cjs
@@ -0,0 +1,93 @@
+'use strict';
+
+/**
+ * #151 — worker inbox-wake watchdog (main-process half). A worker whose pty has
+ * gone quiet while inbox mail sits undrained must be re-woken from MAIN: the
+ * renderer's wake is a single timer with a single-shot dedup, and one missed
+ * tick used to strand the mail until a full app restart.
+ */
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const loadTs = require('./load-ts.cjs');
+
+const { inboxWakeTick, WAKE_QUIET_MS, WAKE_RETRY_MS } = loadTs('src/main/inboxWake.ts');
+
+const QUIET = WAKE_QUIET_MS + 1;
+
+function deps(over = {}) {
+ return {
+ agents: { god: {}, w1: {} },
+ godId: 'god',
+ ptyFor: (id) => (id === 'god' || id === 'w1' ? `pty-${id}` : undefined),
+ hookIdleFor: () => null, // hookless by default — pty recency decides
+ idleFor: () => QUIET,
+ inboxIds: (id) => (id === 'w1' ? ['2026-01-01T00-00-aaa'] : []),
+ ...over
+ };
+}
+
+test('fires for a quiet worker holding undrained mail', () => {
+ const fires = inboxWakeTick(deps(), new Map(), 1_000_000);
+ assert.equal(fires.length, 1);
+ assert.equal(fires[0].agentId, 'w1');
+ assert.equal(fires[0].newestId, '2026-01-01T00-00-aaa');
+ assert.equal(fires[0].count, 1);
+});
+
+test('never fires for god, archived agents, or agents with no live pty', () => {
+ const d = deps({
+ agents: { god: {}, w1: { archived: true }, w2: {} },
+ ptyFor: (id) => (id === 'god' ? 'pty-god' : undefined), // w2 has no pty
+ inboxIds: () => ['mail-1'] // everyone "has" mail — none may fire
+ });
+ assert.equal(inboxWakeTick(d, new Map(), 1_000_000).length, 0);
+});
+
+test('a pty with recent output is mid-turn and is left alone (hookless)', () => {
+ const d = deps({ idleFor: () => WAKE_QUIET_MS - 1 });
+ assert.equal(inboxWakeTick(d, new Map(), 1_000_000).length, 0);
+});
+
+test('hook-idle wins over a constantly-repainting TUI (the claude idle prompt)', () => {
+ // Observed live: an idle claude CLI repaints its prompt continuously, so pty
+ // output recency reads "busy" forever. The Stop hook is the real signal.
+ const d = deps({ hookIdleFor: () => QUIET, idleFor: () => 0 });
+ assert.equal(inboxWakeTick(d, new Map(), 1_000_000).length, 1);
+});
+
+test('hook-busy (a turn in flight or a HITL prompt) never fires, even with a quiet pty', () => {
+ const d = deps({ hookIdleFor: () => 0, idleFor: () => QUIET * 100 });
+ assert.equal(inboxWakeTick(d, new Map(), 1_000_000).length, 0);
+});
+
+test('the same undrained mail re-fires only after the retry window', () => {
+ const state = new Map();
+ const now = 1_000_000;
+ assert.equal(inboxWakeTick(deps(), state, now).length, 1, 'first firing');
+ assert.equal(inboxWakeTick(deps(), state, now + 60_000).length, 0, 'suppressed inside the window');
+ const again = inboxWakeTick(deps(), state, now + WAKE_RETRY_MS + 1);
+ assert.equal(again.length, 1, 're-fires once the window passes — the retry the renderer never had');
+});
+
+test('NEW mail re-fires immediately, even inside the retry window', () => {
+ const state = new Map();
+ const now = 1_000_000;
+ inboxWakeTick(deps(), state, now);
+ const d = deps({ inboxIds: () => ['2026-01-01T00-00-aaa', '2026-01-02T00-00-bbb'] });
+ const fires = inboxWakeTick(d, state, now + 10_000);
+ assert.equal(fires.length, 1);
+ assert.equal(fires[0].newestId, '2026-01-02T00-00-bbb');
+ assert.equal(fires[0].count, 2);
+});
+
+test('a drained inbox clears the per-agent state so later mail fires fresh', () => {
+ const state = new Map();
+ const now = 1_000_000;
+ inboxWakeTick(deps(), state, now);
+ assert.ok(state.has('w1'));
+ inboxWakeTick(deps({ inboxIds: () => [] }), state, now + 10_000);
+ assert.ok(!state.has('w1'), 'drained — forgotten');
+ const fires = inboxWakeTick(deps(), state, now + 20_000);
+ assert.equal(fires.length, 1, 'the same id arriving again is new mail, not a suppressed retry');
+});