Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

feat(realtime): rt-12 completion seam — voice "respond when done" (Realtime Michael P2)

Chaitanya Giri03174b2413c6
diff --git a/src/main/index.ts b/src/main/index.ts
index 07877228..cb05d278 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -29,6 +29,8 @@ import { WebhookServer, type WebhookInbound, type WebhookTaskStatus } from './we
import { transcribeWithGroq, DEFAULT_GROQ_MODEL } from './freeflow';
import { registerRealtimeIpc } from './realtime';
import { registerRealtimeActionIpc } from './realtimeActions';
+import { initCompletionWatcher } from './realtimeCompletionWatcher';
+import type { TaskCard, InboxMessage } from './realtimeCompletionWatcher';
import { TelemetryCollector } from './telemetry';
import { IntegrationBroker } from './integrationBroker';
import * as integrations from './integrations';
@@ -2785,6 +2787,18 @@ registerRealtimeIpc();
// distinct-token rule, the hard allowlist (kill-god / mass-ops forbidden), and the
// michael-voice attribution — lives in ./realtimeActions. This site only injects
// the existing functions; it adds NO new orchestration logic.
+// ─── IPC: Realtime Michael completion watcher (rt-12, Phase 2) ───────────────
+// Jim's net-new engine (realtimeCompletionWatcher.ts) detects a voice-dispatched
+// task finishing (card→done OR a done-reply in michael-voice's inbox) and EMITS it;
+// I own the seam — inject the hive read deps, push completions to the live session
+// (so Michael speaks them unprompted), and bridge waitFor / queue-drain over IPC.
+const completionWatcher = initCompletionWatcher({
+ readTasks: () => { const t = hive.tasks() as { tasks?: TaskCard[] }; return Array.isArray(t?.tasks) ? t.tasks : []; },
+ // Voice dispatches go out as from:michael-voice, so assignee done-replies land here.
+ readInbox: () => { try { return hive.inbox('michael-voice') as unknown as InboxMessage[]; } catch { return []; } },
+ onNotify: (evt) => { try { if (Notification.isSupported()) new Notification({ title: 'Michael', body: evt.summary }).show(); } catch { /* best-effort */ } }
+});
+
registerRealtimeActionIpc({
hiveEnabled: () => hive.enabled(),
hiveSend: (partial, from) => hive.send(partial, from),
@@ -2829,8 +2843,21 @@ registerRealtimeActionIpc({
listMissions: () => readConfig().missions ?? [],
// The spec carries lastFiredAt through from listMissions(), so a wholesale write
// preserves the scheduler's stamps; edit_schedule is deliberate + rare.
- saveMissions: (missions) => { writeConfig({ missions }); }
-});
+ saveMissions: (missions) => { writeConfig({ missions }); },
+ // rt-12: register each voice dispatch so the watcher can detect its completion.
+ trackDispatch: (d) => { try { completionWatcher.track({ ...d, kind: 'dispatch' }); } catch { /* watcher unavailable */ } }
+});
+
+// rt-12 seam: push detected completions to the live floor; bridge live-flag, queue
+// drain (closed-session warm-start), and wait_for over IPC. Then start polling.
+completionWatcher.onCompletion((evt) => { try { liveWebContents()?.send('realtime:completion', evt); } catch { /* window gone */ } });
+ipcMain.handle('realtime:setSessionLive', (_e, live: unknown) => { completionWatcher.setSessionLive(live === true); return { ok: true }; });
+ipcMain.handle('realtime:drainCompletions', () => completionWatcher.drainQueuedCompletions());
+ipcMain.handle('realtime:waitFor', (_e, taskId: unknown, timeoutMs: unknown) =>
+ typeof taskId === 'string'
+ ? completionWatcher.waitFor(taskId, typeof timeoutMs === 'number' && timeoutMs > 0 ? timeoutMs : 120_000)
+ : Promise.resolve({ timedOut: true as const, taskId: '' }));
+completionWatcher.start();
// ─── god-triggered ephemeral Slack workers ──────────────────────────────────
// god drops a spawn-request JSON into HIVE_ROOT/spawn-requests/; MAIN polls that
diff --git a/src/main/realtimeActions.ts b/src/main/realtimeActions.ts
index dc6c6e9a..a1086cac 100644
--- a/src/main/realtimeActions.ts
+++ b/src/main/realtimeActions.ts
@@ -67,6 +67,9 @@ export interface RealtimeActionDeps {
spawnAgent(opts: RealtimeSpawnSpec): Promise<{ ok: boolean; error?: string }>;
listMissions(): ScheduledMission[];
saveMissions(missions: ScheduledMission[]): void;
+ /** rt-12: register a voice dispatch with the completion watcher so the engine can
+ * detect it finishing and speak the notice. Optional — wired in index.ts. */
+ trackDispatch?(d: { correlationId: string; targetAgentId: string; objective?: string; dispatchedAt: number; dispatchMessageId?: string }): void;
}
/** The result every action / confirm / cancel returns to the renderer tool, which
@@ -257,11 +260,13 @@ function execDispatch(deps: RealtimeActionDeps, a: Record<string, unknown>): Act
`CONTEXT: ${str(a.context) || '(none given)'}\n` +
`CONSTRAINTS: ${str(a.constraints) || '(use your judgement; respect the guardrails)'}\n` +
`DONE WHEN: ${str(a.doneWhen) || str(a.done) || 'you report the outcome back to god'}`;
- deps.hiveSend(
+ const msg = deps.hiveSend(
{ to: r.id, act: 'request', subject: `Voice dispatch: ${objective.slice(0, 60)}`, body, requires_reply: true },
VOICE_ACTOR
);
attribute(deps, 'dispatch', r.id, { objective: objective.slice(0, 120) });
+ // rt-12: register so the completion watcher can tell us when r.id finishes.
+ deps.trackDispatch?.({ correlationId: msg.id, targetAgentId: r.id, objective, dispatchedAt: Date.now(), dispatchMessageId: msg.id });
return { ok: true, spoken: `Dispatched to ${r.name}: ${objective.slice(0, 80)}.` };
}
diff --git a/src/preload/index.ts b/src/preload/index.ts
index 4becc3f6..b3747d17 100644
--- a/src/preload/index.ts
+++ b/src/preload/index.ts
@@ -960,7 +960,29 @@ const api = {
): Promise<{ ok: boolean; spoken: string; needsConfirm?: boolean }> =>
ipcRenderer.invoke('realtime:action:confirm', req),
realtimeActionCancel: (): Promise<{ ok: boolean; spoken: string; needsConfirm?: boolean }> =>
- ipcRenderer.invoke('realtime:action:cancel')
+ ipcRenderer.invoke('realtime:action:cancel'),
+ // rt-12 completion seam — a voice-dispatched task finished. `summary` is the
+ // human-speakable line Michael relays; the rest is context for a toast/log.
+ onRealtimeCompletion: (
+ cb: (evt: { correlationId: string; kind: string; targetAgentId: string; taskId?: string; summary: string; completedAt: number; objective?: string }) => void
+ ): (() => void) => {
+ const listener = (_e: IpcRendererEvent, payload: Parameters<typeof cb>[0]) => cb(payload);
+ ipcRenderer.on('realtime:completion', listener);
+ return () => ipcRenderer.removeListener('realtime:completion', listener);
+ },
+ /** Tell main whether a live voice session is open (drives queue-vs-push for completions). */
+ realtimeSetSessionLive: (live: boolean): Promise<{ ok: boolean }> =>
+ ipcRenderer.invoke('realtime:setSessionLive', live),
+ /** Drain completions that arrived while no session was open (warm-start catch-up). */
+ realtimeDrainCompletions: (): Promise<
+ { correlationId: string; kind: string; targetAgentId: string; taskId?: string; summary: string; completedAt: number; objective?: string }[]
+ > => ipcRenderer.invoke('realtime:drainCompletions'),
+ /** Block until a tracked task completes (or times out) — backs the wait_for tool. */
+ realtimeWaitFor: (
+ taskId: string,
+ timeoutMs?: number
+ ): Promise<{ summary: string; targetAgentId: string; taskId?: string } | { timedOut: true; taskId: string }> =>
+ ipcRenderer.invoke('realtime:waitFor', taskId, timeoutMs)
};
contextBridge.exposeInMainWorld('cth', api);
diff --git a/src/renderer/src/realtime/actions.ts b/src/renderer/src/realtime/actions.ts
index ee8fb8cc..5ae8c37f 100644
--- a/src/renderer/src/realtime/actions.ts
+++ b/src/renderer/src/realtime/actions.ts
@@ -146,6 +146,39 @@ export function realtimeActionTools(): ReturnType<typeof tool>[] {
execute: (input) => act('update_task', input)
}),
+ // ── wait for a dispatched task to finish (rt-12 await-in-session) ──────
+ tool({
+ name: 'wait_for',
+ description:
+ 'Wait until a task you dispatched completes, then report it. Use for "tell me when X is done" or "let me know once that finishes". Bounded by a timeout. (For fire-and-forget you do not need this — completions are announced automatically.)',
+ parameters: {
+ type: 'object',
+ properties: {
+ taskId: { type: 'string', description: 'The task id (or dispatch correlation id) to wait on.' },
+ timeoutSeconds: { type: 'number', description: 'Optional max wait in seconds (default 120, max 600).' }
+ },
+ required: ['taskId'],
+ additionalProperties: false
+ },
+ execute: async (input) => {
+ try {
+ const a = obj(input);
+ const taskId = str(a.taskId);
+ if (!taskId) return 'I need a task id to wait on.';
+ const secs = typeof a.timeoutSeconds === 'number' && a.timeoutSeconds > 0 ? Math.min(a.timeoutSeconds, 600) : 120;
+ const res = await window.cth.realtimeWaitFor(taskId, secs * 1000);
+ if (res && 'timedOut' in res && res.timedOut) {
+ return `That one's still running after the wait — I'll let you know the moment it finishes.`;
+ }
+ return (res && 'summary' in res && res.summary) || 'That task completed.';
+ } catch (e) {
+ console.error('[realtime-action] wait_for threw:', e);
+ const msg = e instanceof Error ? e.message : 'an unknown error';
+ return `I couldn't wait on that (${msg}).`;
+ }
+ }
+ }),
+
// ── destructive / expensive (echo-back confirm required) ──────────────
tool({
name: 'spawn_agent',
diff --git a/src/renderer/src/realtime/session.ts b/src/renderer/src/realtime/session.ts
index 6d7ee90d..dfe0ba48 100644
--- a/src/renderer/src/realtime/session.ts
+++ b/src/renderer/src/realtime/session.ts
@@ -96,6 +96,8 @@ let stream: MediaStream | null = null;
let audioEl: HTMLAudioElement | null = null;
/** Guards against overlapping connect() calls racing the async mint/connect. */
let connecting = false;
+/** rt-12: unsubscribe handle for the completion push, active only while a session is live. */
+let offCompletion: (() => void) | null = null;
function setState(patch: Partial<RealtimeMichaelState>): void {
state = { ...state, ...patch };
@@ -270,7 +272,18 @@ export async function connect(): Promise<void> {
const transport = new OpenAIRealtimeWebRTC({ mediaStream: stream, audioElement: audioEl });
// Warm-start: a short, best-effort hive snapshot so Michael's first answer is grounded
// without a tool round-trip (rt-4 realtimeSessionSummary). Returns '' on failure / never throws.
- const warmStart = await realtimeSessionSummary().catch(() => '');
+ let warmStart = await realtimeSessionSummary().catch(() => '');
+ // rt-12: catch up on completions that finished while no session was open, so Michael
+ // can mention them as a "since we last talked" warm-start (the closed-session queue).
+ try {
+ const queued = await window.cth.realtimeDrainCompletions();
+ if (Array.isArray(queued) && queued.length) {
+ const lines = queued.map((c) => c.summary).filter(Boolean).join(' ');
+ if (lines) warmStart = `${warmStart}\nCompletions since you last spoke: ${lines} Mention these to the user when it's natural.`.trim();
+ }
+ } catch {
+ /* warm-start catch-up is best-effort */
+ }
const agent = new RealtimeAgent({
name: 'Michael',
instructions: warmStart
@@ -306,6 +319,20 @@ export async function connect(): Promise<void> {
session = s;
resetRealtimeCost(Date.now()); // rt-9: start the live session cost meter
+ // rt-12: mark the session live (main now pushes completions instead of queuing) and
+ // subscribe so a detected completion makes Michael speak it unprompted.
+ void window.cth.realtimeSetSessionLive(true);
+ offCompletion = window.cth.onRealtimeCompletion((c) => {
+ try {
+ // Feed it as a system-framed notification so the model relays it rather than
+ // treating it as a user request; semantic_vad won't interrupt an active turn.
+ session?.sendMessage(
+ `(System notification — a task you dispatched just finished: ${c.summary}) Briefly let the user know, and offer details if they want them.`
+ );
+ } catch {
+ /* session may be tearing down */
+ }
+ });
setState({
status: 'listening',
muted: false,
@@ -339,6 +366,10 @@ export function disconnect(): void {
session = null;
teardownMedia();
endRealtimeCost(); // rt-9: freeze the session cost meter
+ // rt-12: stop receiving completion pushes; main will queue them until next connect.
+ offCompletion?.();
+ offCompletion = null;
+ void window.cth.realtimeSetSessionLive(false);
// Close the main-process mic gate so the realtime flag doesn't keep the mic permission
// open after we've stopped (fire-and-forget — tracks are already stopped above).
void setMicGate(false);