Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat: dictation endpoint — speech to text for every client

Salman Qureshiee2432ddd541
diff --git a/src/plugins/transcribe.ts b/src/plugins/transcribe.ts
index 1eac01cb..0a29295b 100644
--- a/src/plugins/transcribe.ts
+++ b/src/plugins/transcribe.ts
@@ -1,6 +1,9 @@
/**
- * Speech to text for voice notes, with the OpenAI provider the owner already
- * set up in Mercury (gpt-4o-mini-transcribe, falling back to whisper-1).
+ * Speech to text, with services the owner already set up in Mercury:
+ * 1. the OpenAI provider (gpt-4o-mini-transcribe, falling back to whisper-1),
+ * 2. else a Deepgram key (nova-3) or an ElevenLabs key (scribe_v1) saved
+ * for voice (Settings → Voice / the Optimus plugin).
+ * Used for voice notes from channels and for dictation in the apps.
*/
import { readFileSync, statSync } from 'node:fs';
import { basename } from 'node:path';
@@ -12,26 +15,103 @@ export interface TranscribeCreds {
baseUrl?: string;
}
+export type SpeechEngine =
+ | { kind: 'openai'; label: string; apiKey: string; baseUrl?: string }
+ | { kind: 'deepgram'; label: string; apiKey: string }
+ | { kind: 'elevenlabs'; label: string; apiKey: string };
+
+type Fetch = typeof fetch;
+
export async function transcribeAudio(path: string, creds: TranscribeCreds, signal?: AbortSignal): Promise<string> {
if (statSync(path).size > MAX_BYTES) throw new Error('the voice note is longer than the 25 MB transcription limit');
- const base = (creds.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');
const bytes = readFileSync(path);
+ return transcribeBytes(bytes, 'audio/ogg', basename(path).replace(/\.[^.]+$/, '') + '.ogg', { kind: 'openai', label: 'OpenAI', ...creds }, { signal });
+}
+
+/** File extension a speech API expects for a recorder's mime type. */
+export function audioExtension(mime: string): string {
+ const m = mime.toLowerCase();
+ if (m.includes('webm')) return 'webm';
+ if (m.includes('ogg')) return 'ogg';
+ if (m.includes('mp4') || m.includes('m4a') || m.includes('aac')) return 'm4a';
+ if (m.includes('wav')) return 'wav';
+ if (m.includes('mpeg') || m.includes('mp3')) return 'mp3';
+ return 'webm';
+}
+
+export async function transcribeBytes(
+ bytes: Uint8Array,
+ mime: string,
+ filename: string,
+ engine: SpeechEngine,
+ opts: { signal?: AbortSignal; fetch?: Fetch; language?: string } = {},
+): Promise<string> {
+ if (bytes.byteLength > MAX_BYTES) throw new Error('the recording is longer than the 25 MB transcription limit');
+ if (bytes.byteLength < 200) throw new Error('nothing was recorded');
+ const f = opts.fetch ?? fetch;
+ const signal = opts.signal ?? AbortSignal.timeout(60_000);
+ const type = mime.split(';')[0] || 'audio/webm';
+
+ if (engine.kind === 'deepgram') {
+ const params = new URLSearchParams({ model: 'nova-3', smart_format: 'true', punctuate: 'true' });
+ if (opts.language) params.set('language', opts.language); else params.set('detect_language', 'true');
+ const res = await f(`https://api.deepgram.com/v1/listen?${params}`, {
+ method: 'POST',
+ headers: { Authorization: `Token ${engine.apiKey}`, 'Content-Type': type },
+ body: bytes,
+ signal,
+ });
+ const body = (await res.json().catch(() => ({}))) as { results?: { channels?: Array<{ alternatives?: Array<{ transcript?: string }> }> }; err_msg?: string };
+ if (!res.ok) throw new Error(vendorError('Deepgram', res.status, body.err_msg));
+ return (body.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? '').trim();
+ }
+
+ if (engine.kind === 'elevenlabs') {
+ const form = new FormData();
+ form.append('model_id', 'scribe_v1');
+ if (opts.language) form.append('language_code', opts.language);
+ form.append('file', new Blob([bytes], { type }), filename);
+ const res = await f('https://api.elevenlabs.io/v1/speech-to-text', { method: 'POST', headers: { 'xi-api-key': engine.apiKey }, body: form, signal });
+ const body = (await res.json().catch(() => ({}))) as { text?: string; detail?: { message?: string } | string };
+ if (!res.ok) throw new Error(vendorError('ElevenLabs', res.status, typeof body.detail === 'string' ? body.detail : body.detail?.message));
+ return (body.text ?? '').trim();
+ }
+
+ const base = (engine.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');
let lastError = '';
for (const model of ['gpt-4o-mini-transcribe', 'whisper-1']) {
const form = new FormData();
form.append('model', model);
- form.append('file', new Blob([bytes], { type: 'audio/ogg' }), basename(path).replace(/\.[^.]+$/, '') + '.ogg');
- const res = await fetch(`${base}/audio/transcriptions`, {
- method: 'POST',
- headers: { Authorization: `Bearer ${creds.apiKey}` },
- body: form,
- signal: signal ?? AbortSignal.timeout(60_000),
- });
+ if (opts.language) form.append('language', opts.language);
+ form.append('file', new Blob([bytes], { type }), filename);
+ const res = await f(`${base}/audio/transcriptions`, { method: 'POST', headers: { Authorization: `Bearer ${engine.apiKey}` }, body: form, signal });
const body = (await res.json().catch(() => ({}))) as { text?: string; error?: { message?: string; code?: string } };
if (res.ok && typeof body.text === 'string') return body.text.trim();
- lastError = body.error?.message ?? `HTTP ${res.status}`;
+ lastError = vendorError('OpenAI', res.status, body.error?.message);
// Only a missing model is worth retrying with the other one.
- if (!/model/i.test(lastError) || res.status === 401 || res.status === 429) break;
+ if (!/model/i.test(body.error?.message ?? '') || res.status === 401 || res.status === 429) break;
}
throw new Error(lastError);
}
+
+function vendorError(vendor: string, status: number, detail?: string): string {
+ if (status === 401 || status === 403) return `${vendor} rejected the key. Check it in Settings.`;
+ if (status === 429) return `${vendor} is rate-limiting right now. Try again in a moment.`;
+ return detail ? `${vendor}: ${detail}` : `${vendor} answered ${status}`;
+}
+
+/**
+ * Which speech service to use, from what's configured. OpenAI first (best
+ * at mixed languages and punctuation), then Deepgram, then ElevenLabs.
+ */
+export function pickSpeechEngine(
+ openai: { apiKey?: string; baseUrl?: string; enabled?: boolean } | undefined,
+ secret: (name: string) => string | undefined,
+): SpeechEngine | null {
+ if (openai?.apiKey && openai.enabled !== false) return { kind: 'openai', label: 'OpenAI', apiKey: openai.apiKey, baseUrl: openai.baseUrl };
+ const deepgram = secret('DEEPGRAM_API_KEY');
+ if (deepgram) return { kind: 'deepgram', label: 'Deepgram', apiKey: deepgram };
+ const eleven = secret('ELEVENLABS_API_KEY');
+ if (eleven) return { kind: 'elevenlabs', label: 'ElevenLabs', apiKey: eleven };
+ return null;
+}
diff --git a/src/web/api-contract.ts b/src/web/api-contract.ts
index 5845577e..4d8ec903 100644
--- a/src/web/api-contract.ts
+++ b/src/web/api-contract.ts
@@ -40,6 +40,7 @@ export const API_FEATURES: readonly string[] = Object.freeze([
'chat.stop', // POST /api/chat/stop — interrupt the running turn (same as /stop)
'providers.manage', // GET /api/providers/settings, POST /api/providers/:id/check (live, unsaved), PUT /api/providers/roles; saves apply live; health.setupRequired
'voice.check',
+ 'transcribe', // GET/POST /api/transcribe — dictation: raw audio in, text out
'cloud.signin', // GET /api/cloud/status, POST/GET/DELETE /api/cloud/signin (browser pairing from any client) // POST /api/voice/check { vendor, key?, endpoint? }
'brain.map', // GET /api/brain/map[?archive=1] — themes, sub-themes, notes, related links
]);
diff --git a/src/web/api/transcribe-route.test.ts b/src/web/api/transcribe-route.test.ts
new file mode 100644
index 00000000..c34354b7
--- /dev/null
+++ b/src/web/api/transcribe-route.test.ts
@@ -0,0 +1,69 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+const home = await vi.hoisted(async () => {
+ const fs = await import('node:fs');
+ const os = await import('node:os');
+ const path = await import('node:path');
+ for (const k of ['OPENAI_API_KEY', 'DEEPGRAM_API_KEY', 'ELEVENLABS_API_KEY']) delete process.env[k];
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-transcribe-'));
+ fs.writeFileSync(path.join(dir, 'mercury.yaml'), 'identity:\n owner: Test\n');
+ process.env.MERCURY_HOME = dir;
+ return dir;
+});
+
+import transcribe, { setTranscribeFetch } from './transcribe.js';
+
+const audio = new Uint8Array(4000).fill(7);
+const post = (body: Uint8Array = audio, type = 'audio/webm;codecs=opus', q = '') =>
+ transcribe.request(`http://127.0.0.1/api/transcribe${q}`, { method: 'POST', headers: { 'Content-Type': type }, body });
+const withOpenAI = () => writeFileSync(join(home, 'mercury.yaml'), [
+ 'identity:', ' owner: Test', 'providers:', ' openai:', ' name: openai', ' apiKey: sk-test-openai-000000000000', ' baseUrl: https://api.openai.com/v1', ' model: gpt-4o-mini', ' enabled: true', '',
+].join('\n'));
+
+afterEach(() => { setTranscribeFetch(undefined); delete process.env.DEEPGRAM_API_KEY; });
+
+describe('dictation', () => {
+ it('with no speech service, says how to turn one on', async () => {
+ expect(await (await transcribe.request('http://127.0.0.1/api/transcribe')).json()).toEqual({ available: false, engine: null });
+ const res = await post();
+ expect(res.status).toBe(412);
+ expect(await res.json()).toMatchObject({ needs: 'speech' });
+ });
+
+ it('falls back to a Deepgram voice key', async () => {
+ process.env.DEEPGRAM_API_KEY = 'dg-key';
+ const f = vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ results: { channels: [{ alternatives: [{ transcript: 'hello from deepgram' }] }] } }) });
+ setTranscribeFetch(f as never);
+ const body = await (await post()).json() as any;
+ expect(body).toEqual({ text: 'hello from deepgram', engine: 'Deepgram' });
+ expect(f.mock.calls[0][0]).toContain('api.deepgram.com/v1/listen');
+ expect(f.mock.calls[0][1].headers['Content-Type']).toBe('audio/webm');
+ });
+
+ it('prefers the OpenAI provider, sends the recording as a file, and passes the language', async () => {
+ withOpenAI();
+ process.env.DEEPGRAM_API_KEY = 'dg-key';
+ const f = vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ text: ' Remind me to call Heer. ' }) });
+ setTranscribeFetch(f as never);
+ const body = await (await post(audio, 'audio/mp4', '?language=en')).json() as any;
+ expect(body).toEqual({ text: 'Remind me to call Heer.', engine: 'OpenAI' });
+ const form = f.mock.calls[0][1].body as FormData;
+ expect(form.get('model')).toBe('gpt-4o-mini-transcribe');
+ expect(form.get('language')).toBe('en');
+ expect((form.get('file') as File).name).toBe('dictation.m4a');
+ });
+
+ it('a rejected key and an empty recording read plainly', async () => {
+ withOpenAI();
+ setTranscribeFetch(vi.fn().mockResolvedValue({ ok: false, status: 401, json: async () => ({ error: { message: 'bad key' } }) }) as never);
+ expect((await (await post()).json() as any).error).toMatch(/rejected the key/);
+ expect((await (await post(new Uint8Array(10))).json() as any).error).toMatch(/Didn't catch anything/);
+ });
+
+ it('refuses non-audio bodies', async () => {
+ withOpenAI();
+ expect((await post(audio, 'application/json')).status).toBe(415);
+ });
+});
diff --git a/src/web/api/transcribe.ts b/src/web/api/transcribe.ts
new file mode 100644
index 00000000..20f515fa
--- /dev/null
+++ b/src/web/api/transcribe.ts
@@ -0,0 +1,59 @@
+import { Hono } from 'hono';
+import { loadConfig } from '../../utils/config.js';
+import { secretValue } from '../../plugins/store.js';
+import { audioExtension, pickSpeechEngine, transcribeBytes } from '../../plugins/transcribe.js';
+
+/**
+ * Dictation for every client: speak instead of typing.
+ *
+ * GET /api/transcribe { available, engine } — which speech service would be used
+ * POST /api/transcribe[?language=] raw audio body (audio/webm, audio/mp4, …) → { text, engine }
+ *
+ * Uses a service the owner already set up (OpenAI provider, else a Deepgram
+ * or ElevenLabs voice key). With none, 412 { needs: 'speech' } so the app can
+ * say how to turn it on. Audio is sent once and not kept.
+ */
+
+const MAX_BYTES = 25 * 1024 * 1024;
+
+type Fetch = typeof fetch;
+let doFetch: Fetch | undefined;
+/** Test seam. */
+export function setTranscribeFetch(f: Fetch | undefined): void { doFetch = f; }
+
+function engine() {
+ return pickSpeechEngine(loadConfig().providers.openai, secretValue);
+}
+
+const transcribe = new Hono();
+
+transcribe.get('/api/transcribe', (c) => {
+ const e = engine();
+ return c.json({ available: !!e, engine: e?.label ?? null });
+});
+
+transcribe.post('/api/transcribe', async (c) => {
+ const e = engine();
+ if (!e) {
+ return c.json({ error: 'Voice typing needs a speech service: an OpenAI key (Settings → Models), or a Deepgram or ElevenLabs key (Settings → Voice).', needs: 'speech' }, 412);
+ }
+ const declared = Number(c.req.header('content-length') ?? 0);
+ if (declared > MAX_BYTES) return c.json({ error: 'That recording is too long (over 25 MB). Keep dictation under about 20 minutes.' }, 413);
+ const mime = (c.req.header('content-type') ?? 'audio/webm').toLowerCase();
+ if (!mime.startsWith('audio/') && !mime.startsWith('video/webm') && mime !== 'application/octet-stream') {
+ return c.json({ error: 'Send the recording as audio.' }, 415);
+ }
+ const bytes = new Uint8Array(await c.req.arrayBuffer());
+ if (bytes.byteLength > MAX_BYTES) return c.json({ error: 'That recording is too long (over 25 MB).' }, 413);
+ const language = c.req.query('language');
+ const lang = language && /^[a-z]{2,3}(-[A-Za-z]{2,4})?$/.test(language) ? language : undefined;
+ try {
+ const text = await transcribeBytes(bytes, mime, `dictation.${audioExtension(mime)}`, e, { fetch: doFetch, language: lang });
+ return c.json({ text, engine: e.label });
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ return c.json({ error: message === 'nothing was recorded' ? "Didn't catch anything. Check the microphone and try again." : `Couldn't transcribe: ${message}`, engine: e.label }, 502);
+ }
+});
+
+export default transcribe;
diff --git a/src/web/server.ts b/src/web/server.ts
index 463aaa60..7548063a 100644
--- a/src/web/server.ts
+++ b/src/web/server.ts
@@ -13,6 +13,7 @@ import authRoutes from './api/auth.js';
import statusRoutes, { updateStatus } from './api/status.js';
import providerRoutes from './api/providers.js';
import cloudRoutes from './api/cloud.js';
+import transcribeRoutes from './api/transcribe.js';
import configRoutes from './api/config.js';
import systemRoutes, { setScheduler } from './api/system.js';
import brainRoutes, { setUserMemory } from './api/brain.js';
@@ -93,6 +94,7 @@ app.route('/', authRoutes);
app.route('/', statusRoutes);
app.route('/', providerRoutes);
app.route('/', cloudRoutes);
+app.route('/', transcribeRoutes);
app.route('/', configRoutes);
app.route('/', systemRoutes);
app.route('/', brainRoutes);