Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat(voice): start calls from clients, live transcript and levels, calls saved as chats
Salman Qureshi81a3a6791145
diff --git a/src/index.ts b/src/index.tsindex 0eac6f35..85af4313 100644--- a/src/index.ts+++ b/src/index.ts@@ -98,6 +98,7 @@ import { startWebServer, stopWebServer, updateStatus as updateWebStatus, setUserimport { validateApiKey, validateBaseUrl, validateModelName } from './providers/provider-meta.js';import { setFirstProviderHandler, setProvidersReload } from './web/api/providers.js';import { spawnDetachedRestart } from './cli/daemon.js';+import { recordVoiceCalls } from './plugins/voice-calls.js';import { isWebAuthInitialized, setWebPassword, writeAttachToken, initWebAuth } from './web/auth.js';import { startLoopLagMonitor } from './utils/loop-lag.js';import { McpManager, probeMcpServer, validateMcpConfig, type McpServerStatus } from './capabilities/mcp/index.js';@@ -2502,6 +2503,8 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {migrateLegacyMemory();const shortTerm = new ShortTermMemory(config);const sessions = new SessionRepository();+ // Desktop voice calls are saved as chat threads, live.+ recordVoiceCalls(sessions);setWebSessions(sessions);const longTerm = new LongTermMemory(config);const episodic = new EpisodicMemory(config);diff --git a/src/plugins/channel-bridge.ts b/src/plugins/channel-bridge.tsindex fa633fa4..babfd639 100644--- a/src/plugins/channel-bridge.ts+++ b/src/plugins/channel-bridge.ts@@ -279,6 +279,15 @@ export class PluginChannel extends BaseChannel {else if (method === 'mercury/channel/message') this.onMessageNotification(params as IncomingParams);else if (method === 'mercury/call/ended') this.onCallEnded(params);else if (method === 'mercury/channel/activity') this.onActivity(params);+ else if (method === 'mercury/voice/transcript' && this.desktop) {+ // Optimus 0.5+: each thing said on a desktop session, as it's said.+ const p = (params ?? {}) as Record<string, unknown>;+ const role = p.role === 'owner' ? 'owner' : p.role === 'mercury' ? 'mercury' : null;+ if (role && typeof p.text === 'string') voiceFeed.said(String(p.callId ?? ''), role, p.text.slice(0, 4000));+ } else if (method === 'mercury/voice/level' && this.desktop) {+ const p = (params ?? {}) as Record<string, unknown>;+ voiceFeed.levels(Number(p.mic ?? 0), Number(p.out ?? 0));+ }}async handleRequest(method: string, params: unknown): Promise<unknown> {diff --git a/src/plugins/voice-calls.test.ts b/src/plugins/voice-calls.test.tsnew file mode 100644index 00000000..2f308c16--- /dev/null+++ b/src/plugins/voice-calls.test.ts@@ -0,0 +1,86 @@+import { mkdtempSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { SessionRepository } from '../sessions/index.js';+import { voiceFeed } from './voice-feed.js';+import { callDuration, recordVoiceCalls } from './voice-calls.js';++let sessions: SessionRepository;+let stop: () => void;++beforeEach(() => {+ voiceFeed.reset();+ sessions = new SessionRepository({ rootDir: mkdtempSync(join(tmpdir(), 'mercury-calls-')), autoMigrate: false });+ stop = recordVoiceCalls(sessions);+});+afterEach(() => stop());++describe('voice calls as chat threads', () => {+ it('writes what was said into a thread as it happens, and names it with the length at the end', () => {+ voiceFeed.started('desk-1');+ const threadId = voiceFeed.snapshot().threadId!;+ expect(threadId).toBeTruthy();+ voiceFeed.said('desk-1', 'owner', 'What is on my calendar today?');+ voiceFeed.asked('desk-1', 'Read the owner\'s calendar for today');+ voiceFeed.said('desk-1', 'mercury', 'You have two meetings.');+ voiceFeed.answered('Two meetings: 10:00 Design review, 15:00 Investor call.');++ const live = sessions.get(threadId);+ expect(live.messages.map((m) => [m.role, m.metadata?.voiceRole])).toEqual([+ ['user', 'owner'], ['assistant', 'mercury'], ['assistant', 'result'],+ ]);+ expect(live.messages[0].content).toBe('What is on my calendar today?');+ expect(live.title).toMatch(/^Voice call · /);++ voiceFeed.ended('desk-1', 'completed');+ expect(sessions.get(threadId).title).toMatch(/^Voice call · .+ · \d+s$/);+ });++ it('a call where nothing was said leaves no thread', () => {+ voiceFeed.started('desk-2');+ const threadId = voiceFeed.snapshot().threadId!;+ voiceFeed.ended('desk-2', 'failed: no microphone');+ expect(sessions.list().some((s) => s.id === threadId)).toBe(false);+ });++ it('without a live transcript (older Optimus), the question and answer are what gets saved', () => {+ voiceFeed.started('desk-3');+ const threadId = voiceFeed.snapshot().threadId!;+ voiceFeed.asked('desk-3', 'Open Gmail');+ voiceFeed.answered('Opened Gmail.');+ expect(sessions.get(threadId).messages.map((m) => [m.role, m.content])).toEqual([['user', 'Open Gmail'], ['assistant', 'Opened Gmail.']]);+ });+});++describe('voice feed controls and levels', () => {+ it('start only works with the plugin attached and no session running', () => {+ const sent = vi.fn();+ expect(voiceFeed.control('start')).toBe(false);+ voiceFeed.attach('Optimus', sent);+ expect(voiceFeed.control('start')).toBe(true);+ expect(sent).toHaveBeenCalledWith('start');+ expect(voiceFeed.control('mute')).toBe(false);+ voiceFeed.started('desk-4');+ expect(voiceFeed.control('start')).toBe(false);+ expect(voiceFeed.control('mute')).toBe(true);+ });++ it('streams levels while a session runs, clamped, and never stores them', () => {+ const seen: unknown[] = [];+ const off = voiceFeed.subscribe((e) => { if (e.type === 'level') seen.push(e.levels); });+ voiceFeed.levels(0.5, 0.2);+ expect(seen).toHaveLength(0);+ voiceFeed.started('desk-5');+ voiceFeed.levels(1.7, Number.NaN);+ off();+ expect(seen).toEqual([expect.objectContaining({ mic: 1, out: 0 })]);+ expect(JSON.stringify(voiceFeed.snapshot())).not.toContain('mic');+ });++ it('formats call lengths', () => {+ expect(callDuration(42)).toBe('42s');+ expect(callDuration(185)).toBe('3 min');+ expect(callDuration(3720)).toBe('1h 2m');+ });+});diff --git a/src/plugins/voice-calls.ts b/src/plugins/voice-calls.tsnew file mode 100644index 00000000..3dcce8d8--- /dev/null+++ b/src/plugins/voice-calls.ts@@ -0,0 +1,65 @@+import type { SessionRepository } from '../sessions/index.js';+import { voiceFeed, type VoiceRole } from './voice-feed.js';+import { logger } from '../utils/logger.js';++/**+ * Every desktop voice call is also a regular chat thread, written as it+ * happens: what the owner said, what Mercury said, and the full answers+ * behind them. Clients open it like any thread (and see it fill in live);+ * a call that ends with nothing said leaves no thread behind.+ */++const SAVED: Partial<Record<VoiceRole, 'user' | 'assistant'>> = { owner: 'user', mercury: 'assistant', result: 'assistant', approval: 'assistant' };++function clock(at: number): string {+ return new Date(at).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' });+}++export function callDuration(secs: number): string {+ if (secs < 60) return `${Math.max(1, Math.round(secs))}s`;+ const m = Math.floor(secs / 60);+ return m < 60 ? `${m} min` : `${Math.floor(m / 60)}h ${m % 60}m`;+}++export function recordVoiceCalls(sessions: SessionRepository): () => void {+ let current: { callId: string; threadId: string; startedAt: number; saved: number } | null = null;++ return voiceFeed.subscribe((ev) => {+ try {+ const s = ev.state;+ if (ev.type === 'state') {+ if (s.active && s.callId && current?.callId !== s.callId) {+ const startedAt = s.startedAt ?? Date.now();+ const session = sessions.create({ title: `Voice call · ${clock(startedAt)}`, titleSource: 'user' });+ current = { callId: s.callId, threadId: session.id, startedAt, saved: 0 };+ voiceFeed.setThread(session.id);+ } else if (!s.active && current && s.endedAt) {+ const done = current;+ current = null;+ if (done.saved === 0) {+ // Nothing was said (a failed start, a press by accident): no empty thread in the list.+ sessions.markDeleted(done.threadId);+ return;+ }+ const secs = (s.endedAt - done.startedAt) / 1000;+ sessions.updateTitle(done.threadId, `Voice call · ${clock(done.startedAt)} · ${callDuration(secs)}`, 'user');+ }+ return;+ }+ if (ev.type === 'turn' && ev.turn && current) {+ const role = SAVED[ev.turn.role];+ if (!role) return;+ const content = ev.turn.role === 'approval' ? `Asked to go ahead: ${ev.turn.text}` : ev.turn.text;+ sessions.appendMessage(current.threadId, {+ role,+ content,+ timestamp: ev.turn.at,+ metadata: { voiceCall: current.callId, voiceRole: ev.turn.role },+ });+ current.saved += 1;+ }+ } catch (err) {+ logger.warn({ err: err instanceof Error ? err.message : String(err) }, 'Could not save the voice call to its chat thread');+ }+ });+}diff --git a/src/plugins/voice-feed.ts b/src/plugins/voice-feed.tsindex d5078081..3ea3fd44 100644--- a/src/plugins/voice-feed.ts+++ b/src/plugins/voice-feed.ts@@ -11,7 +11,12 @@* already kept by the session store like any other turn.*/-export type VoiceRole = 'owner' | 'mercury' | 'activity' | 'approval' | 'system';+/**+ * owner: what the owner said · mercury: what Mercury said out loud ·+ * result: Mercury's full answer behind what it said (details, links) ·+ * activity: what it's doing · approval: what it's asking to do.+ */+export type VoiceRole = 'owner' | 'mercury' | 'result' | 'activity' | 'approval' | 'system';export interface VoiceTurn {id: number;@@ -35,21 +40,28 @@ export interface VoiceState {thinking: boolean;activity: string | null;outcome: string | null;+ /** The chat thread this call is saved to, live (see voice-calls.ts). */+ threadId: string | null;turns: VoiceTurn[];}-export type VoiceAction = 'mute' | 'unmute' | 'interrupt' | 'stop';-export const VOICE_ACTIONS: readonly VoiceAction[] = ['mute', 'unmute', 'interrupt', 'stop'];+/** How loud the owner (mic) and Mercury (speaker) are right now, 0-1. Not stored; streamed ~12×/s. */+export interface VoiceLevels { mic: number; out: number; at: number }++export type VoiceAction = 'start' | 'mute' | 'unmute' | 'interrupt' | 'stop';+export const VOICE_ACTIONS: readonly VoiceAction[] = ['start', 'mute', 'unmute', 'interrupt', 'stop'];-type Listener = (event: { type: 'state' | 'turn'; state: VoiceState; turn?: VoiceTurn }) => void;+type Listener = (event: { type: 'state' | 'turn' | 'level'; state: VoiceState; turn?: VoiceTurn; levels?: VoiceLevels }) => void;const MAX_TURNS = 200;class VoiceFeed {private state: VoiceState = {available: false, plugin: null, active: false, callId: null, startedAt: null, endedAt: null,- listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, turns: [],+ listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, threadId: null, turns: [],};+ /** The plugin streams what's said as it's said (Optimus 0.5+); older ones only report questions and answers. */+ private live = false;private seq = 0;private listeners = new Set<Listener>();private controller: ((action: VoiceAction) => void) | null = null;@@ -75,15 +87,37 @@ class VoiceFeed {this.patch({ available: false, listening: false, speaking: false, working: false, muted: false, thinking: false, active: false });}+ /** 'start' needs a plugin and no session running; the rest need a running session. */control(action: VoiceAction): boolean {- if (!this.controller || !this.state.active) return false;+ if (!this.controller) return false;+ if (action === 'start' ? this.state.active : !this.state.active) return false;this.controller(action);return true;}started(callId: string): void {if (this.state.active && this.state.callId === callId) return;- this.patch({ active: true, callId, startedAt: Date.now(), endedAt: null, outcome: null, thinking: false, activity: null, turns: [] });+ this.live = false;+ this.patch({ active: true, callId, startedAt: Date.now(), endedAt: null, outcome: null, thinking: false, activity: null, threadId: null, turns: [] });+ }++ /** What was said out loud, as it's said: the owner, or Mercury's voice. */+ said(callId: string, role: 'owner' | 'mercury', text: string): void {+ const t = text.trim();+ if (!t) return;+ if (callId) this.started(callId);+ this.live = true;+ this.push(role, t);+ }++ levels(mic: number, out: number): void {+ if (!this.state.active) return;+ const clamp = (n: number) => (Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : 0);+ this.emit({ type: 'level', state: this.state, levels: { mic: clamp(mic), out: clamp(out), at: Date.now() } });+ }++ setThread(threadId: string): void {+ this.patch({ threadId });}activity(p: { listening: boolean; working: boolean; muted: boolean; speaking?: boolean }): void {@@ -96,7 +130,9 @@ class VoiceFeed {asked(callId: string, question: string): void {this.started(callId);this.patch({ thinking: true, activity: null });- this.push('owner', question);+ // With a live transcript the owner's own words are already there; this is the request it became.+ if (this.live) this.push('activity', `Asked Mercury: ${question}`);+ else this.push('owner', question);}doing(activity: string): void {@@ -108,7 +144,8 @@ class VoiceFeed {answered(answer: string): void {if (!this.state.active) return;this.patch({ thinking: false, activity: null });- this.push('mercury', answer);+ // Live: the voice speaks a summary on its own; keep the full answer as the detail behind it.+ this.push(this.live ? 'result' : 'mercury', answer);}approval(prompt: string): void {@@ -125,7 +162,8 @@ class VoiceFeed {reset(): void {this.controller = null;this.seq = 0;- this.state = { ...this.state, available: false, plugin: null, active: false, callId: null, startedAt: null, endedAt: null, listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, turns: [] };+ this.live = false;+ this.state = { ...this.state, available: false, plugin: null, active: false, callId: null, startedAt: null, endedAt: null, listening: false, speaking: false, working: false, muted: false, thinking: false, activity: null, outcome: null, threadId: null, turns: [] };}private push(role: VoiceRole, text: string): void {diff --git a/src/web/api/chat.ts b/src/web/api/chat.tsindex 633f569c..f793a2f9 100644--- a/src/web/api/chat.ts+++ b/src/web/api/chat.ts@@ -35,6 +35,8 @@ function toThread(session: ReturnType<SessionRepository['get']>, includeMessagesshortId: session.shortId,alias: session.alias,title: session.title,+ // A desktop voice call saved as a thread (voice-calls.ts).+ voiceCall: typeof session.messages[0]?.metadata?.voiceCall === 'string',createdAt: new Date(session.createdAt).toISOString(),updatedAt: new Date(session.updatedAt).toISOString(),messages: includeMessages ? session.messages.map((message) => ({@@ -45,6 +47,7 @@ function toThread(session: ReturnType<SessionRepository['get']>, includeMessagestimestamp: new Date(message.timestamp).toISOString(),kind: message.kind,sequence: message.sequence,+ ...(typeof message.metadata?.voiceRole === 'string' ? { voice: message.metadata.voiceRole } : {}),...(Array.isArray(message.metadata?.attachments) && message.metadata.attachments.length > 0? { attachments: (message.metadata.attachments as Array<{ name: string; kind: string; mime?: string; ref: string }>).map((a) => ({ name: a.name, kind: a.kind, mime: a.mime, url: `/api/chat/uploads/${a.ref}` })) }: {}),diff --git a/src/web/api/voice.ts b/src/web/api/voice.tsindex b6d06377..3235aca9 100644--- a/src/web/api/voice.ts+++ b/src/web/api/voice.ts@@ -7,7 +7,7 @@ import { secretValue } from '../../plugins/store.js';* Live voice (Optimus) for clients:* GET /api/voice current session state and transcript* GET /api/voice/events SSE: `state` and `turn` events (snapshot first)- * POST /api/voice/control { action: mute | unmute | interrupt | stop }+ * POST /api/voice/control { action: start | mute | unmute | interrupt | stop }* Owner-authenticated like every /api route.*/const voice = new Hono();@@ -36,6 +36,7 @@ voice.get('/api/voice/events', (c) => {send('state', voiceFeed.snapshot());const unsubscribe = voiceFeed.subscribe((ev) => {if (ev.type === 'turn' && ev.turn) send('turn', ev.turn);+ else if (ev.type === 'level' && ev.levels) send('level', ev.levels);// Turns travel on their own; state events stay small.else send('state', { ...ev.state, turns: undefined });});@@ -53,7 +54,11 @@ voice.post('/api/voice/control', async (c) => {const body = await c.req.json<{ action?: unknown }>().catch(() => ({} as { action?: unknown }));const action = body.action as VoiceAction;if (!VOICE_ACTIONS.includes(action)) return c.json({ error: `action must be one of ${VOICE_ACTIONS.join(', ')}` }, 400);- if (!voiceFeed.control(action)) return c.json({ error: 'No voice session is running' }, 409);+ if (!voiceFeed.control(action)) {+ const s = voiceFeed.snapshot();+ const why = !s.available ? 'Voice needs the Optimus plugin, installed and turned on.' : action === 'start' ? 'A voice session is already running.' : 'No voice session is running';+ return c.json({ error: why }, 409);+ }return c.json({ ok: true });});