an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
feat(realtime): rt-12 completion seam — voice "respond when done" (Realtime Michael P2)
Chaitanya Giri03174b2413c6
diff --git a/src/main/index.ts b/src/main/index.tsindex 07877228..cb05d278 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -29,6 +29,8 @@ import { WebhookServer, type WebhookInbound, type WebhookTaskStatus } from './weimport { transcribeWithGroq, DEFAULT_GROQ_MODEL } from './freeflow';import { registerRealtimeIpc } from './realtime';import { registerRealtimeActionIpc } from './realtimeActions';+import { initCompletionWatcher } from './realtimeCompletionWatcher';+import type { TaskCard, InboxMessage } from './realtimeCompletionWatcher';import { TelemetryCollector } from './telemetry';import { IntegrationBroker } from './integrationBroker';import * as integrations from './integrations';@@ -2785,6 +2787,18 @@ registerRealtimeIpc();// distinct-token rule, the hard allowlist (kill-god / mass-ops forbidden), and the// michael-voice attribution — lives in ./realtimeActions. This site only injects// the existing functions; it adds NO new orchestration logic.+// ─── IPC: Realtime Michael completion watcher (rt-12, Phase 2) ───────────────+// Jim's net-new engine (realtimeCompletionWatcher.ts) detects a voice-dispatched+// task finishing (card→done OR a done-reply in michael-voice's inbox) and EMITS it;+// I own the seam — inject the hive read deps, push completions to the live session+// (so Michael speaks them unprompted), and bridge waitFor / queue-drain over IPC.+const completionWatcher = initCompletionWatcher({+ readTasks: () => { const t = hive.tasks() as { tasks?: TaskCard[] }; return Array.isArray(t?.tasks) ? t.tasks : []; },+ // Voice dispatches go out as from:michael-voice, so assignee done-replies land here.+ readInbox: () => { try { return hive.inbox('michael-voice') as unknown as InboxMessage[]; } catch { return []; } },+ onNotify: (evt) => { try { if (Notification.isSupported()) new Notification({ title: 'Michael', body: evt.summary }).show(); } catch { /* best-effort */ } }+});+registerRealtimeActionIpc({hiveEnabled: () => hive.enabled(),hiveSend: (partial, from) => hive.send(partial, from),@@ -2829,8 +2843,21 @@ registerRealtimeActionIpc({listMissions: () => readConfig().missions ?? [],// The spec carries lastFiredAt through from listMissions(), so a wholesale write// preserves the scheduler's stamps; edit_schedule is deliberate + rare.- saveMissions: (missions) => { writeConfig({ missions }); }-});+ saveMissions: (missions) => { writeConfig({ missions }); },+ // rt-12: register each voice dispatch so the watcher can detect its completion.+ trackDispatch: (d) => { try { completionWatcher.track({ ...d, kind: 'dispatch' }); } catch { /* watcher unavailable */ } }+});++// rt-12 seam: push detected completions to the live floor; bridge live-flag, queue+// drain (closed-session warm-start), and wait_for over IPC. Then start polling.+completionWatcher.onCompletion((evt) => { try { liveWebContents()?.send('realtime:completion', evt); } catch { /* window gone */ } });+ipcMain.handle('realtime:setSessionLive', (_e, live: unknown) => { completionWatcher.setSessionLive(live === true); return { ok: true }; });+ipcMain.handle('realtime:drainCompletions', () => completionWatcher.drainQueuedCompletions());+ipcMain.handle('realtime:waitFor', (_e, taskId: unknown, timeoutMs: unknown) =>+ typeof taskId === 'string'+ ? completionWatcher.waitFor(taskId, typeof timeoutMs === 'number' && timeoutMs > 0 ? timeoutMs : 120_000)+ : Promise.resolve({ timedOut: true as const, taskId: '' }));+completionWatcher.start();// ─── god-triggered ephemeral Slack workers ──────────────────────────────────// god drops a spawn-request JSON into HIVE_ROOT/spawn-requests/; MAIN polls thatdiff --git a/src/main/realtimeActions.ts b/src/main/realtimeActions.tsindex dc6c6e9a..a1086cac 100644--- a/src/main/realtimeActions.ts+++ b/src/main/realtimeActions.ts@@ -67,6 +67,9 @@ export interface RealtimeActionDeps {spawnAgent(opts: RealtimeSpawnSpec): Promise<{ ok: boolean; error?: string }>;listMissions(): ScheduledMission[];saveMissions(missions: ScheduledMission[]): void;+ /** rt-12: register a voice dispatch with the completion watcher so the engine can+ * detect it finishing and speak the notice. Optional — wired in index.ts. */+ trackDispatch?(d: { correlationId: string; targetAgentId: string; objective?: string; dispatchedAt: number; dispatchMessageId?: string }): void;}/** The result every action / confirm / cancel returns to the renderer tool, which@@ -257,11 +260,13 @@ function execDispatch(deps: RealtimeActionDeps, a: Record<string, unknown>): Act`CONTEXT: ${str(a.context) || '(none given)'}\n` +`CONSTRAINTS: ${str(a.constraints) || '(use your judgement; respect the guardrails)'}\n` +`DONE WHEN: ${str(a.doneWhen) || str(a.done) || 'you report the outcome back to god'}`;- deps.hiveSend(+ const msg = deps.hiveSend({ to: r.id, act: 'request', subject: `Voice dispatch: ${objective.slice(0, 60)}`, body, requires_reply: true },VOICE_ACTOR);attribute(deps, 'dispatch', r.id, { objective: objective.slice(0, 120) });+ // rt-12: register so the completion watcher can tell us when r.id finishes.+ deps.trackDispatch?.({ correlationId: msg.id, targetAgentId: r.id, objective, dispatchedAt: Date.now(), dispatchMessageId: msg.id });return { ok: true, spoken: `Dispatched to ${r.name}: ${objective.slice(0, 80)}.` };}diff --git a/src/preload/index.ts b/src/preload/index.tsindex 4becc3f6..b3747d17 100644--- a/src/preload/index.ts+++ b/src/preload/index.ts@@ -960,7 +960,29 @@ const api = {): Promise<{ ok: boolean; spoken: string; needsConfirm?: boolean }> =>ipcRenderer.invoke('realtime:action:confirm', req),realtimeActionCancel: (): Promise<{ ok: boolean; spoken: string; needsConfirm?: boolean }> =>- ipcRenderer.invoke('realtime:action:cancel')+ ipcRenderer.invoke('realtime:action:cancel'),+ // rt-12 completion seam — a voice-dispatched task finished. `summary` is the+ // human-speakable line Michael relays; the rest is context for a toast/log.+ onRealtimeCompletion: (+ cb: (evt: { correlationId: string; kind: string; targetAgentId: string; taskId?: string; summary: string; completedAt: number; objective?: string }) => void+ ): (() => void) => {+ const listener = (_e: IpcRendererEvent, payload: Parameters<typeof cb>[0]) => cb(payload);+ ipcRenderer.on('realtime:completion', listener);+ return () => ipcRenderer.removeListener('realtime:completion', listener);+ },+ /** Tell main whether a live voice session is open (drives queue-vs-push for completions). */+ realtimeSetSessionLive: (live: boolean): Promise<{ ok: boolean }> =>+ ipcRenderer.invoke('realtime:setSessionLive', live),+ /** Drain completions that arrived while no session was open (warm-start catch-up). */+ realtimeDrainCompletions: (): Promise<+ { correlationId: string; kind: string; targetAgentId: string; taskId?: string; summary: string; completedAt: number; objective?: string }[]+ > => ipcRenderer.invoke('realtime:drainCompletions'),+ /** Block until a tracked task completes (or times out) — backs the wait_for tool. */+ realtimeWaitFor: (+ taskId: string,+ timeoutMs?: number+ ): Promise<{ summary: string; targetAgentId: string; taskId?: string } | { timedOut: true; taskId: string }> =>+ ipcRenderer.invoke('realtime:waitFor', taskId, timeoutMs)};contextBridge.exposeInMainWorld('cth', api);diff --git a/src/renderer/src/realtime/actions.ts b/src/renderer/src/realtime/actions.tsindex ee8fb8cc..5ae8c37f 100644--- a/src/renderer/src/realtime/actions.ts+++ b/src/renderer/src/realtime/actions.ts@@ -146,6 +146,39 @@ export function realtimeActionTools(): ReturnType<typeof tool>[] {execute: (input) => act('update_task', input)}),+ // ── wait for a dispatched task to finish (rt-12 await-in-session) ──────+ tool({+ name: 'wait_for',+ description:+ 'Wait until a task you dispatched completes, then report it. Use for "tell me when X is done" or "let me know once that finishes". Bounded by a timeout. (For fire-and-forget you do not need this — completions are announced automatically.)',+ parameters: {+ type: 'object',+ properties: {+ taskId: { type: 'string', description: 'The task id (or dispatch correlation id) to wait on.' },+ timeoutSeconds: { type: 'number', description: 'Optional max wait in seconds (default 120, max 600).' }+ },+ required: ['taskId'],+ additionalProperties: false+ },+ execute: async (input) => {+ try {+ const a = obj(input);+ const taskId = str(a.taskId);+ if (!taskId) return 'I need a task id to wait on.';+ const secs = typeof a.timeoutSeconds === 'number' && a.timeoutSeconds > 0 ? Math.min(a.timeoutSeconds, 600) : 120;+ const res = await window.cth.realtimeWaitFor(taskId, secs * 1000);+ if (res && 'timedOut' in res && res.timedOut) {+ return `That one's still running after the wait — I'll let you know the moment it finishes.`;+ }+ return (res && 'summary' in res && res.summary) || 'That task completed.';+ } catch (e) {+ console.error('[realtime-action] wait_for threw:', e);+ const msg = e instanceof Error ? e.message : 'an unknown error';+ return `I couldn't wait on that (${msg}).`;+ }+ }+ }),+// ── destructive / expensive (echo-back confirm required) ──────────────tool({name: 'spawn_agent',diff --git a/src/renderer/src/realtime/session.ts b/src/renderer/src/realtime/session.tsindex 6d7ee90d..dfe0ba48 100644--- a/src/renderer/src/realtime/session.ts+++ b/src/renderer/src/realtime/session.ts@@ -96,6 +96,8 @@ let stream: MediaStream | null = null;let audioEl: HTMLAudioElement | null = null;/** Guards against overlapping connect() calls racing the async mint/connect. */let connecting = false;+/** rt-12: unsubscribe handle for the completion push, active only while a session is live. */+let offCompletion: (() => void) | null = null;function setState(patch: Partial<RealtimeMichaelState>): void {state = { ...state, ...patch };@@ -270,7 +272,18 @@ export async function connect(): Promise<void> {const transport = new OpenAIRealtimeWebRTC({ mediaStream: stream, audioElement: audioEl });// Warm-start: a short, best-effort hive snapshot so Michael's first answer is grounded// without a tool round-trip (rt-4 realtimeSessionSummary). Returns '' on failure / never throws.- const warmStart = await realtimeSessionSummary().catch(() => '');+ let warmStart = await realtimeSessionSummary().catch(() => '');+ // rt-12: catch up on completions that finished while no session was open, so Michael+ // can mention them as a "since we last talked" warm-start (the closed-session queue).+ try {+ const queued = await window.cth.realtimeDrainCompletions();+ if (Array.isArray(queued) && queued.length) {+ const lines = queued.map((c) => c.summary).filter(Boolean).join(' ');+ if (lines) warmStart = `${warmStart}\nCompletions since you last spoke: ${lines} Mention these to the user when it's natural.`.trim();+ }+ } catch {+ /* warm-start catch-up is best-effort */+ }const agent = new RealtimeAgent({name: 'Michael',instructions: warmStart@@ -306,6 +319,20 @@ export async function connect(): Promise<void> {session = s;resetRealtimeCost(Date.now()); // rt-9: start the live session cost meter+ // rt-12: mark the session live (main now pushes completions instead of queuing) and+ // subscribe so a detected completion makes Michael speak it unprompted.+ void window.cth.realtimeSetSessionLive(true);+ offCompletion = window.cth.onRealtimeCompletion((c) => {+ try {+ // Feed it as a system-framed notification so the model relays it rather than+ // treating it as a user request; semantic_vad won't interrupt an active turn.+ session?.sendMessage(+ `(System notification — a task you dispatched just finished: ${c.summary}) Briefly let the user know, and offer details if they want them.`+ );+ } catch {+ /* session may be tearing down */+ }+ });setState({status: 'listening',muted: false,@@ -339,6 +366,10 @@ export function disconnect(): void {session = null;teardownMedia();endRealtimeCost(); // rt-9: freeze the session cost meter+ // rt-12: stop receiving completion pushes; main will queue them until next connect.+ offCompletion?.();+ offCompletion = null;+ void window.cth.realtimeSetSessionLive(false);// Close the main-process mic gate so the realtime flag doesn't keep the mic permission// open after we've stopped (fire-and-forget — tracks are already stopped above).void setMicGate(false);