Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat(voice): start calls from clients, live transcript and levels, calls saved as chats

Salman Qureshi81a3a6791145
diff --git a/src/index.ts b/src/index.ts
index 0eac6f35..85af4313 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -98,6 +98,7 @@ import { startWebServer, stopWebServer, updateStatus as updateWebStatus, setUser
import { validateApiKey, validateBaseUrl, validateModelName } from './providers/provider-meta.js';
import { setFirstProviderHandler, setProvidersReload } from './web/api/providers.js';
import { spawnDetachedRestart } from './cli/daemon.js';
+import { recordVoiceCalls } from './plugins/voice-calls.js';
import { isWebAuthInitialized, setWebPassword, writeAttachToken, initWebAuth } from './web/auth.js';
import { startLoopLagMonitor } from './utils/loop-lag.js';
import { McpManager, probeMcpServer, validateMcpConfig, type McpServerStatus } from './capabilities/mcp/index.js';
@@ -2502,6 +2503,8 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {
migrateLegacyMemory();
const shortTerm = new ShortTermMemory(config);
const sessions = new SessionRepository();
+ // Desktop voice calls are saved as chat threads, live.
+ recordVoiceCalls(sessions);
setWebSessions(sessions);
const longTerm = new LongTermMemory(config);
const episodic = new EpisodicMemory(config);
diff --git a/src/plugins/channel-bridge.ts b/src/plugins/channel-bridge.ts
index fa633fa4..babfd639 100644
--- a/src/plugins/channel-bridge.ts
+++ b/src/plugins/channel-bridge.ts
@@ -279,6 +279,15 @@ export class PluginChannel extends BaseChannel {
else if (method === 'mercury/channel/message') this.onMessageNotification(params as IncomingParams);
else if (method === 'mercury/call/ended') this.onCallEnded(params);
else if (method === 'mercury/channel/activity') this.onActivity(params);
+ else if (method === 'mercury/voice/transcript' && this.desktop) {
+ // Optimus 0.5+: each thing said on a desktop session, as it's said.
+ const p = (params ?? {}) as Record<string, unknown>;
+ const role = p.role === 'owner' ? 'owner' : p.role === 'mercury' ? 'mercury' : null;
+ if (role && typeof p.text === 'string') voiceFeed.said(String(p.callId ?? ''), role, p.text.slice(0, 4000));
+ } else if (method === 'mercury/voice/level' && this.desktop) {
+ const p = (params ?? {}) as Record<string, unknown>;
+ voiceFeed.levels(Number(p.mic ?? 0), Number(p.out ?? 0));
+ }
}
async handleRequest(method: string, params: unknown): Promise<unknown> {
diff --git a/src/plugins/voice-calls.test.ts b/src/plugins/voice-calls.test.ts
new file mode 100644
index 00000000..2f308c16
--- /dev/null
+++ b/src/plugins/voice-calls.test.ts
@@ -0,0 +1,86 @@
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { SessionRepository } from '../sessions/index.js';
+import { voiceFeed } from './voice-feed.js';
+import { callDuration, recordVoiceCalls } from './voice-calls.js';
+
+let sessions: SessionRepository;
+let stop: () => void;
+
+beforeEach(() => {
+ voiceFeed.reset();
+ sessions = new SessionRepository({ rootDir: mkdtempSync(join(tmpdir(), 'mercury-calls-')), autoMigrate: false });
+ stop = recordVoiceCalls(sessions);
+});
+afterEach(() => stop());
+
+describe('voice calls as chat threads', () => {
+ it('writes what was said into a thread as it happens, and names it with the length at the end', () => {
+ voiceFeed.started('desk-1');
+ const threadId = voiceFeed.snapshot().threadId!;
+ expect(threadId).toBeTruthy();
+ voiceFeed.said('desk-1', 'owner', 'What is on my calendar today?');
+ voiceFeed.asked('desk-1', 'Read the owner\'s calendar for today');
+ voiceFeed.said('desk-1', 'mercury', 'You have two meetings.');
+ voiceFeed.answered('Two meetings: 10:00 Design review, 15:00 Investor call.');
+
+ const live = sessions.get(threadId);
+ expect(live.messages.map((m) => [m.role, m.metadata?.voiceRole])).toEqual([
+ ['user', 'owner'], ['assistant', 'mercury'], ['assistant', 'result'],
+ ]);
+ expect(live.messages[0].content).toBe('What is on my calendar today?');
+ expect(live.title).toMatch(/^Voice call · /);
+
+ voiceFeed.ended('desk-1', 'completed');
+ expect(sessions.get(threadId).title).toMatch(/^Voice call · .+ · \d+s$/);
+ });
+
+ it('a call where nothing was said leaves no thread', () => {
+ voiceFeed.started('desk-2');
+ const threadId = voiceFeed.snapshot().threadId!;
+ voiceFeed.ended('desk-2', 'failed: no microphone');
+ expect(sessions.list().some((s) => s.id === threadId)).toBe(false);
+ });
+
+ it('without a live transcript (older Optimus), the question and answer are what gets saved', () => {
+ voiceFeed.started('desk-3');
+ const threadId = voiceFeed.snapshot().threadId!;
+ voiceFeed.asked('desk-3', 'Open Gmail');
+ voiceFeed.answered('Opened Gmail.');
+ expect(sessions.get(threadId).messages.map((m) => [m.role, m.content])).toEqual([['user', 'Open Gmail'], ['assistant', 'Opened Gmail.']]);
+ });
+});
+
+describe('voice feed controls and levels', () => {
+ it('start only works with the plugin attached and no session running', () => {
+ const sent = vi.fn();
+ expect(voiceFeed.control('start')).toBe(false);
+ voiceFeed.attach('Optimus', sent);
+ expect(voiceFeed.control('start')).toBe(true);
+ expect(sent).toHaveBeenCalledWith('start');
+ expect(voiceFeed.control('mute')).toBe(false);
+ voiceFeed.started('desk-4');
+ expect(voiceFeed.control('start')).toBe(false);
+ expect(voiceFeed.control('mute')).toBe(true);
+ });
+
+ it('streams levels while a session runs, clamped, and never stores them', () => {
+ const seen: unknown[] = [];
+ const off = voiceFeed.subscribe((e) => { if (e.type === 'level') seen.push(e.levels); });
+ voiceFeed.levels(0.5, 0.2);
+ expect(seen).toHaveLength(0);
+ voiceFeed.started('desk-5');
+ voiceFeed.levels(1.7, Number.NaN);
+ off();
+ expect(seen).toEqual([expect.objectContaining({ mic: 1, out: 0 })]);
+ expect(JSON.stringify(voiceFeed.snapshot())).not.toContain('mic');
+ });
+
+ it('formats call lengths', () => {
+ expect(callDuration(42)).toBe('42s');
+ expect(callDuration(185)).toBe('3 min');
+ expect(callDuration(3720)).toBe('1h 2m');
+ });
+});
diff --git a/src/plugins/voice-calls.ts b/src/plugins/voice-calls.ts
new file mode 100644
index 00000000..3dcce8d8
--- /dev/null
+++ b/src/plugins/voice-calls.ts
@@ -0,0 +1,65 @@
+import type { SessionRepository } from '../sessions/index.js';
+import { voiceFeed, type VoiceRole } from './voice-feed.js';
+import { logger } from '../utils/logger.js';
+
+/**
+ * Every desktop voice call is also a regular chat thread, written as it
+ * happens: what the owner said, what Mercury said, and the full answers
+ * behind them. Clients open it like any thread (and see it fill in live);
+ * a call that ends with nothing said leaves no thread behind.
+ */
+
+const SAVED: Partial<Record<VoiceRole, 'user' | 'assistant'>> = { owner: 'user', mercury: 'assistant', result: 'assistant', approval: 'assistant' };
+
+function clock(at: number): string {
+ return new Date(at).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' });
+}
+
+export function callDuration(secs: number): string {
+ if (secs < 60) return `${Math.max(1, Math.round(secs))}s`;
+ const m = Math.floor(secs / 60);
+ return m < 60 ? `${m} min` : `${Math.floor(m / 60)}h ${m % 60}m`;
+}
+
+export function recordVoiceCalls(sessions: SessionRepository): () => void {
+ let current: { callId: string; threadId: string; startedAt: number; saved: number } | null = null;
+
+ return voiceFeed.subscribe((ev) => {
+ try {
+ const s = ev.state;
+ if (ev.type === 'state') {
+ if (s.active && s.callId && current?.callId !== s.callId) {
+ const startedAt = s.startedAt ?? Date.now();
+ const session = sessions.create({ title: `Voice call · ${clock(startedAt)}`, titleSource: 'user' });
+ current = { callId: s.callId, threadId: session.id, startedAt, saved: 0 };
+ voiceFeed.setThread(session.id);
+ } else if (!s.active && current && s.endedAt) {
+ const done = current;
+ current = null;
+ if (done.saved === 0) {
+ // Nothing was said (a failed start, a press by accident): no empty thread in the list.
+ sessions.markDeleted(done.threadId);
+ return;
+ }
+ const secs = (s.endedAt - done.startedAt) / 1000;
+ sessions.updateTitle(done.threadId, `Voice call · ${clock(done.startedAt)} · ${callDuration(secs)}`, 'user');
+ }
+ return;
+ }
+ if (ev.type === 'turn' && ev.turn && current) {
+ const role = SAVED[ev.turn.role];
+ if (!role) return;
+ const content = ev.turn.role === 'approval' ? `Asked to go ahead: ${ev.turn.text}` : ev.turn.text;
+ sessions.appendMessage(current.threadId, {
+ role,
+ content,
+ timestamp: ev.turn.at,
+ metadata: { voiceCall: current.callId, voiceRole: ev.turn.role },
+ });
+ current.saved += 1;
+ }
+ } catch (err) {
+ logger.warn({ err: err instanceof Error ? err.message : String(err) }, 'Could not save the voice call to its chat thread');
+ }
+ });
+}
diff --git a/src/plugins/voice-feed.ts b/src/plugins/voice-feed.ts
index d5078081..3ea3fd44 100644
--- a/src/plugins/voice-feed.ts
+++ b/src/plugins/voice-feed.ts
@@ -11,7 +11,12 @@
* already kept by the session store like any other turn.
*/
-export type VoiceRole = 'owner' | 'mercury' | 'activity' | 'approval' | 'system';
+/**
+ * owner: what the owner said · mercury: what Mercury said out loud ·
+ * result: Mercury's full answer behind what it said (details, links) ·
+ * activity: what it's doing · approval: what it's asking to do.
+ */
+export type VoiceRole = 'owner' | 'mercury' | 'result' | 'activity' | 'approval' | 'system';
export interface VoiceTurn {
id: number;
@@ -35,21 +40,28 @@ export interface VoiceState {
thinking: boolean;
activity: string | null;
outcome: string | null;
+ /** The chat thread this call is saved to, live (see voice-calls.ts). */
+ threadId: string | null;
turns: VoiceTurn[];
}
-export type VoiceAction = 'mute' | 'unmute' | 'interrupt' | 'stop';
-export const VOICE_ACTIONS: readonly VoiceAction[] = ['mute', 'unmute', 'interrupt', 'stop'];
+/** How loud the owner (mic) and Mercury (speaker) are right now, 0-1. Not stored; streamed ~12×/s. */
+export interface VoiceLevels { mic: number; out: number; at: number }
+
+export type VoiceAction = 'start' | 'mute' | 'unmute' | 'interrupt' | 'stop';
+export const VOICE_ACTIONS: readonly VoiceAction[] = ['start', 'mute', 'unmute', 'interrupt', 'stop'];
-type Listener = (event: { type: 'state' | 'turn'; state: VoiceState; turn?: VoiceTurn }) => void;
+type Listener = (event: { type: 'state' | 'turn' | 'level'; state: VoiceState; turn?: VoiceTurn; levels?: VoiceLevels }) => void;
const MAX_TURNS = 200;
class VoiceFeed {
private state: VoiceState = {
available: false, plugin: null, active: false, callId: null, startedAt: null, endedAt: null,
- listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, turns: [],
+ listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, threadId: null, turns: [],
};
+ /** The plugin streams what's said as it's said (Optimus 0.5+); older ones only report questions and answers. */
+ private live = false;
private seq = 0;
private listeners = new Set<Listener>();
private controller: ((action: VoiceAction) => void) | null = null;
@@ -75,15 +87,37 @@ class VoiceFeed {
this.patch({ available: false, listening: false, speaking: false, working: false, muted: false, thinking: false, active: false });
}
+ /** 'start' needs a plugin and no session running; the rest need a running session. */
control(action: VoiceAction): boolean {
- if (!this.controller || !this.state.active) return false;
+ if (!this.controller) return false;
+ if (action === 'start' ? this.state.active : !this.state.active) return false;
this.controller(action);
return true;
}
started(callId: string): void {
if (this.state.active && this.state.callId === callId) return;
- this.patch({ active: true, callId, startedAt: Date.now(), endedAt: null, outcome: null, thinking: false, activity: null, turns: [] });
+ this.live = false;
+ this.patch({ active: true, callId, startedAt: Date.now(), endedAt: null, outcome: null, thinking: false, activity: null, threadId: null, turns: [] });
+ }
+
+ /** What was said out loud, as it's said: the owner, or Mercury's voice. */
+ said(callId: string, role: 'owner' | 'mercury', text: string): void {
+ const t = text.trim();
+ if (!t) return;
+ if (callId) this.started(callId);
+ this.live = true;
+ this.push(role, t);
+ }
+
+ levels(mic: number, out: number): void {
+ if (!this.state.active) return;
+ const clamp = (n: number) => (Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : 0);
+ this.emit({ type: 'level', state: this.state, levels: { mic: clamp(mic), out: clamp(out), at: Date.now() } });
+ }
+
+ setThread(threadId: string): void {
+ this.patch({ threadId });
}
activity(p: { listening: boolean; working: boolean; muted: boolean; speaking?: boolean }): void {
@@ -96,7 +130,9 @@ class VoiceFeed {
asked(callId: string, question: string): void {
this.started(callId);
this.patch({ thinking: true, activity: null });
- this.push('owner', question);
+ // With a live transcript the owner's own words are already there; this is the request it became.
+ if (this.live) this.push('activity', `Asked Mercury: ${question}`);
+ else this.push('owner', question);
}
doing(activity: string): void {
@@ -108,7 +144,8 @@ class VoiceFeed {
answered(answer: string): void {
if (!this.state.active) return;
this.patch({ thinking: false, activity: null });
- this.push('mercury', answer);
+ // Live: the voice speaks a summary on its own; keep the full answer as the detail behind it.
+ this.push(this.live ? 'result' : 'mercury', answer);
}
approval(prompt: string): void {
@@ -125,7 +162,8 @@ class VoiceFeed {
reset(): void {
this.controller = null;
this.seq = 0;
- this.state = { ...this.state, available: false, plugin: null, active: false, callId: null, startedAt: null, endedAt: null, listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, turns: [] };
+ this.live = false;
+ this.state = { ...this.state, available: false, plugin: null, active: false, callId: null, startedAt: null, endedAt: null, listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, threadId: null, turns: [] };
}
private push(role: VoiceRole, text: string): void {
diff --git a/src/web/api/chat.ts b/src/web/api/chat.ts
index 633f569c..f793a2f9 100644
--- a/src/web/api/chat.ts
+++ b/src/web/api/chat.ts
@@ -35,6 +35,8 @@ function toThread(session: ReturnType<SessionRepository['get']>, includeMessages
shortId: session.shortId,
alias: session.alias,
title: session.title,
+ // A desktop voice call saved as a thread (voice-calls.ts).
+ voiceCall: typeof session.messages[0]?.metadata?.voiceCall === 'string',
createdAt: new Date(session.createdAt).toISOString(),
updatedAt: new Date(session.updatedAt).toISOString(),
messages: includeMessages ? session.messages.map((message) => ({
@@ -45,6 +47,7 @@ function toThread(session: ReturnType<SessionRepository['get']>, includeMessages
timestamp: new Date(message.timestamp).toISOString(),
kind: message.kind,
sequence: message.sequence,
+ ...(typeof message.metadata?.voiceRole === 'string' ? { voice: message.metadata.voiceRole } : {}),
...(Array.isArray(message.metadata?.attachments) && message.metadata.attachments.length > 0
? { attachments: (message.metadata.attachments as Array<{ name: string; kind: string; mime?: string; ref: string }>).map((a) => ({ name: a.name, kind: a.kind, mime: a.mime, url: `/api/chat/uploads/${a.ref}` })) }
: {}),
diff --git a/src/web/api/voice.ts b/src/web/api/voice.ts
index b6d06377..3235aca9 100644
--- a/src/web/api/voice.ts
+++ b/src/web/api/voice.ts
@@ -7,7 +7,7 @@ import { secretValue } from '../../plugins/store.js';
* Live voice (Optimus) for clients:
* GET /api/voice current session state and transcript
* GET /api/voice/events SSE: `state` and `turn` events (snapshot first)
- * POST /api/voice/control { action: mute | unmute | interrupt | stop }
+ * POST /api/voice/control { action: start | mute | unmute | interrupt | stop }
* Owner-authenticated like every /api route.
*/
const voice = new Hono();
@@ -36,6 +36,7 @@ voice.get('/api/voice/events', (c) => {
send('state', voiceFeed.snapshot());
const unsubscribe = voiceFeed.subscribe((ev) => {
if (ev.type === 'turn' && ev.turn) send('turn', ev.turn);
+ else if (ev.type === 'level' && ev.levels) send('level', ev.levels);
// Turns travel on their own; state events stay small.
else send('state', { ...ev.state, turns: undefined });
});
@@ -53,7 +54,11 @@ voice.post('/api/voice/control', async (c) => {
const body = await c.req.json<{ action?: unknown }>().catch(() => ({} as { action?: unknown }));
const action = body.action as VoiceAction;
if (!VOICE_ACTIONS.includes(action)) return c.json({ error: `action must be one of ${VOICE_ACTIONS.join(', ')}` }, 400);
- if (!voiceFeed.control(action)) return c.json({ error: 'No voice session is running' }, 409);
+ if (!voiceFeed.control(action)) {
+ const s = voiceFeed.snapshot();
+ const why = !s.available ? 'Voice needs the Optimus plugin, installed and turned on.' : action === 'start' ? 'A voice session is already running.' : 'No voice session is running';
+ return c.json({ error: why }, 409);
+ }
return c.json({ ok: true });
});