Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat: dictation endpoint — speech to text for every client
Salman Qureshiee2432ddd541
diff --git a/src/plugins/transcribe.ts b/src/plugins/transcribe.tsindex 1eac01cb..0a29295b 100644--- a/src/plugins/transcribe.ts+++ b/src/plugins/transcribe.ts@@ -1,6 +1,9 @@/**- * Speech to text for voice notes, with the OpenAI provider the owner already- * set up in Mercury (gpt-4o-mini-transcribe, falling back to whisper-1).+ * Speech to text, with services the owner already set up in Mercury:+ * 1. the OpenAI provider (gpt-4o-mini-transcribe, falling back to whisper-1),+ * 2. else a Deepgram key (nova-3) or an ElevenLabs key (scribe_v1) saved+ * for voice (Settings → Voice / the Optimus plugin).+ * Used for voice notes from channels and for dictation in the apps.*/import { readFileSync, statSync } from 'node:fs';import { basename } from 'node:path';@@ -12,26 +15,103 @@ export interface TranscribeCreds {baseUrl?: string;}+export type SpeechEngine =+ | { kind: 'openai'; label: string; apiKey: string; baseUrl?: string }+ | { kind: 'deepgram'; label: string; apiKey: string }+ | { kind: 'elevenlabs'; label: string; apiKey: string };++type Fetch = typeof fetch;+export async function transcribeAudio(path: string, creds: TranscribeCreds, signal?: AbortSignal): Promise<string> {if (statSync(path).size > MAX_BYTES) throw new Error('the voice note is longer than the 25 MB transcription limit');- const base = (creds.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');const bytes = readFileSync(path);+ return transcribeBytes(bytes, 'audio/ogg', basename(path).replace(/\.[^.]+$/, '') + '.ogg', { kind: 'openai', label: 'OpenAI', ...creds }, { signal });+}++/** File extension a speech API expects for a recorder's mime type. */+export function audioExtension(mime: string): string {+ const m = mime.toLowerCase();+ if (m.includes('webm')) return 'webm';+ if (m.includes('ogg')) return 'ogg';+ if (m.includes('mp4') || m.includes('m4a') || m.includes('aac')) return 'm4a';+ if (m.includes('wav')) return 'wav';+ if (m.includes('mpeg') || m.includes('mp3')) return 'mp3';+ return 'webm';+}++export async function transcribeBytes(+ bytes: Uint8Array,+ mime: string,+ filename: string,+ engine: SpeechEngine,+ opts: { signal?: AbortSignal; fetch?: Fetch; language?: string } = {},+): Promise<string> {+ if (bytes.byteLength > MAX_BYTES) throw new Error('the recording is longer than the 25 MB transcription limit');+ if (bytes.byteLength < 200) throw new Error('nothing was recorded');+ const f = opts.fetch ?? fetch;+ const signal = opts.signal ?? AbortSignal.timeout(60_000);+ const type = mime.split(';')[0] || 'audio/webm';++ if (engine.kind === 'deepgram') {+ const params = new URLSearchParams({ model: 'nova-3', smart_format: 'true', punctuate: 'true' });+ if (opts.language) params.set('language', opts.language); else params.set('detect_language', 'true');+ const res = await f(`https://api.deepgram.com/v1/listen?${params}`, {+ method: 'POST',+ headers: { Authorization: `Token ${engine.apiKey}`, 'Content-Type': type },+ body: bytes,+ signal,+ });+ const body = (await res.json().catch(() => ({}))) as { results?: { channels?: Array<{ alternatives?: Array<{ transcript?: string }> }> }; err_msg?: string };+ if (!res.ok) throw new Error(vendorError('Deepgram', res.status, body.err_msg));+ return (body.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? '').trim();+ }++ if (engine.kind === 'elevenlabs') {+ const form = new FormData();+ form.append('model_id', 'scribe_v1');+ if (opts.language) form.append('language_code', opts.language);+ form.append('file', new Blob([bytes], { type }), filename);+ const res = await f('https://api.elevenlabs.io/v1/speech-to-text', { method: 'POST', headers: { 'xi-api-key': engine.apiKey }, body: form, signal });+ const body = (await res.json().catch(() => ({}))) as { text?: string; detail?: { message?: string } | string };+ if (!res.ok) throw new Error(vendorError('ElevenLabs', res.status, typeof body.detail === 'string' ? body.detail : body.detail?.message));+ return (body.text ?? '').trim();+ }++ const base = (engine.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');let lastError = '';for (const model of ['gpt-4o-mini-transcribe', 'whisper-1']) {const form = new FormData();form.append('model', model);- form.append('file', new Blob([bytes], { type: 'audio/ogg' }), basename(path).replace(/\.[^.]+$/, '') + '.ogg');- const res = await fetch(`${base}/audio/transcriptions`, {- method: 'POST',- headers: { Authorization: `Bearer ${creds.apiKey}` },- body: form,- signal: signal ?? AbortSignal.timeout(60_000),- });+ if (opts.language) form.append('language', opts.language);+ form.append('file', new Blob([bytes], { type }), filename);+ const res = await f(`${base}/audio/transcriptions`, { method: 'POST', headers: { Authorization: `Bearer ${engine.apiKey}` }, body: form, signal });const body = (await res.json().catch(() => ({}))) as { text?: string; error?: { message?: string; code?: string } };if (res.ok && typeof body.text === 'string') return body.text.trim();- lastError = body.error?.message ?? `HTTP ${res.status}`;+ lastError = vendorError('OpenAI', res.status, body.error?.message);// Only a missing model is worth retrying with the other one.- if (!/model/i.test(lastError) || res.status === 401 || res.status === 429) break;+ if (!/model/i.test(body.error?.message ?? '') || res.status === 401 || res.status === 429) break;}throw new Error(lastError);}++function vendorError(vendor: string, status: number, detail?: string): string {+ if (status === 401 || status === 403) return `${vendor} rejected the key. Check it in Settings.`;+ if (status === 429) return `${vendor} is rate-limiting right now. Try again in a moment.`;+ return detail ? `${vendor}: ${detail}` : `${vendor} answered ${status}`;+}++/**+ * Which speech service to use, from what's configured. OpenAI first (best+ * at mixed languages and punctuation), then Deepgram, then ElevenLabs.+ */+export function pickSpeechEngine(+ openai: { apiKey?: string; baseUrl?: string; enabled?: boolean } | undefined,+ secret: (name: string) => string | undefined,+): SpeechEngine | null {+ if (openai?.apiKey && openai.enabled !== false) return { kind: 'openai', label: 'OpenAI', apiKey: openai.apiKey, baseUrl: openai.baseUrl };+ const deepgram = secret('DEEPGRAM_API_KEY');+ if (deepgram) return { kind: 'deepgram', label: 'Deepgram', apiKey: deepgram };+ const eleven = secret('ELEVENLABS_API_KEY');+ if (eleven) return { kind: 'elevenlabs', label: 'ElevenLabs', apiKey: eleven };+ return null;+}diff --git a/src/web/api-contract.ts b/src/web/api-contract.tsindex 5845577e..4d8ec903 100644--- a/src/web/api-contract.ts+++ b/src/web/api-contract.ts@@ -40,6 +40,7 @@ export const API_FEATURES: readonly string[] = Object.freeze(['chat.stop', // POST /api/chat/stop — interrupt the running turn (same as /stop)'providers.manage', // GET /api/providers/settings, POST /api/providers/:id/check (live, unsaved), PUT /api/providers/roles; saves apply live; health.setupRequired'voice.check',+ 'transcribe', // GET/POST /api/transcribe — dictation: raw audio in, text out'cloud.signin', // GET /api/cloud/status, POST/GET/DELETE /api/cloud/signin (browser pairing from any client) // POST /api/voice/check { vendor, key?, endpoint? }'brain.map', // GET /api/brain/map[?archive=1] — themes, sub-themes, notes, related links]);diff --git a/src/web/api/transcribe-route.test.ts b/src/web/api/transcribe-route.test.tsnew file mode 100644index 00000000..c34354b7--- /dev/null+++ b/src/web/api/transcribe-route.test.ts@@ -0,0 +1,69 @@+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, describe, expect, it, vi } from 'vitest';++const home = await vi.hoisted(async () => {+ const fs = await import('node:fs');+ const os = await import('node:os');+ const path = await import('node:path');+ for (const k of ['OPENAI_API_KEY', 'DEEPGRAM_API_KEY', 'ELEVENLABS_API_KEY']) delete process.env[k];+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-transcribe-'));+ fs.writeFileSync(path.join(dir, 'mercury.yaml'), 'identity:\n owner: Test\n');+ process.env.MERCURY_HOME = dir;+ return dir;+});++import transcribe, { setTranscribeFetch } from './transcribe.js';++const audio = new Uint8Array(4000).fill(7);+const post = (body: Uint8Array = audio, type = 'audio/webm;codecs=opus', q = '') =>+ transcribe.request(`http://127.0.0.1/api/transcribe${q}`, { method: 'POST', headers: { 'Content-Type': type }, body });+const withOpenAI = () => writeFileSync(join(home, 'mercury.yaml'), [+ 'identity:', ' owner: Test', 'providers:', ' openai:', ' name: openai', ' apiKey: sk-test-openai-000000000000', ' baseUrl: https://api.openai.com/v1', ' model: gpt-4o-mini', ' enabled: true', '',+].join('\n'));++afterEach(() => { setTranscribeFetch(undefined); delete process.env.DEEPGRAM_API_KEY; });++describe('dictation', () => {+ it('with no speech service, says how to turn one on', async () => {+ expect(await (await transcribe.request('http://127.0.0.1/api/transcribe')).json()).toEqual({ available: false, engine: null });+ const res = await post();+ expect(res.status).toBe(412);+ expect(await res.json()).toMatchObject({ needs: 'speech' });+ });++ it('falls back to a Deepgram voice key', async () => {+ process.env.DEEPGRAM_API_KEY = 'dg-key';+ const f = vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ results: { channels: [{ alternatives: [{ transcript: 'hello from deepgram' }] }] } }) });+ setTranscribeFetch(f as never);+ const body = await (await post()).json() as any;+ expect(body).toEqual({ text: 'hello from deepgram', engine: 'Deepgram' });+ expect(f.mock.calls[0][0]).toContain('api.deepgram.com/v1/listen');+ expect(f.mock.calls[0][1].headers['Content-Type']).toBe('audio/webm');+ });++ it('prefers the OpenAI provider, sends the recording as a file, and passes the language', async () => {+ withOpenAI();+ process.env.DEEPGRAM_API_KEY = 'dg-key';+ const f = vi.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ text: ' Remind me to call Heer. ' }) });+ setTranscribeFetch(f as never);+ const body = await (await post(audio, 'audio/mp4', '?language=en')).json() as any;+ expect(body).toEqual({ text: 'Remind me to call Heer.', engine: 'OpenAI' });+ const form = f.mock.calls[0][1].body as FormData;+ expect(form.get('model')).toBe('gpt-4o-mini-transcribe');+ expect(form.get('language')).toBe('en');+ expect((form.get('file') as File).name).toBe('dictation.m4a');+ });++ it('a rejected key and an empty recording read plainly', async () => {+ withOpenAI();+ setTranscribeFetch(vi.fn().mockResolvedValue({ ok: false, status: 401, json: async () => ({ error: { message: 'bad key' } }) }) as never);+ expect((await (await post()).json() as any).error).toMatch(/rejected the key/);+ expect((await (await post(new Uint8Array(10))).json() as any).error).toMatch(/Didn't catch anything/);+ });++ it('refuses non-audio bodies', async () => {+ withOpenAI();+ expect((await post(audio, 'application/json')).status).toBe(415);+ });+});diff --git a/src/web/api/transcribe.ts b/src/web/api/transcribe.tsnew file mode 100644index 00000000..20f515fa--- /dev/null+++ b/src/web/api/transcribe.ts@@ -0,0 +1,59 @@+import { Hono } from 'hono';+import { loadConfig } from '../../utils/config.js';+import { secretValue } from '../../plugins/store.js';+import { audioExtension, pickSpeechEngine, transcribeBytes } from '../../plugins/transcribe.js';++/**+ * Dictation for every client: speak instead of typing.+ *+ * GET /api/transcribe { available, engine } — which speech service would be used+ * POST /api/transcribe[?language=] raw audio body (audio/webm, audio/mp4, …) → { text, engine }+ *+ * Uses a service the owner already set up (OpenAI provider, else a Deepgram+ * or ElevenLabs voice key). With none, 412 { needs: 'speech' } so the app can+ * say how to turn it on. Audio is sent once and not kept.+ */++const MAX_BYTES = 25 * 1024 * 1024;++type Fetch = typeof fetch;+let doFetch: Fetch | undefined;+/** Test seam. */+export function setTranscribeFetch(f: Fetch | undefined): void { doFetch = f; }++function engine() {+ return pickSpeechEngine(loadConfig().providers.openai, secretValue);+}++const transcribe = new Hono();++transcribe.get('/api/transcribe', (c) => {+ const e = engine();+ return c.json({ available: !!e, engine: e?.label ?? null });+});++transcribe.post('/api/transcribe', async (c) => {+ const e = engine();+ if (!e) {+ return c.json({ error: 'Voice typing needs a speech service: an OpenAI key (Settings → Models), or a Deepgram or ElevenLabs key (Settings → Voice).', needs: 'speech' }, 412);+ }+ const declared = Number(c.req.header('content-length') ?? 0);+ if (declared > MAX_BYTES) return c.json({ error: 'That recording is too long (over 25 MB). Keep dictation under about 20 minutes.' }, 413);+ const mime = (c.req.header('content-type') ?? 'audio/webm').toLowerCase();+ if (!mime.startsWith('audio/') && !mime.startsWith('video/webm') && mime !== 'application/octet-stream') {+ return c.json({ error: 'Send the recording as audio.' }, 415);+ }+ const bytes = new Uint8Array(await c.req.arrayBuffer());+ if (bytes.byteLength > MAX_BYTES) return c.json({ error: 'That recording is too long (over 25 MB).' }, 413);+ const language = c.req.query('language');+ const lang = language && /^[a-z]{2,3}(-[A-Za-z]{2,4})?$/.test(language) ? language : undefined;+ try {+ const text = await transcribeBytes(bytes, mime, `dictation.${audioExtension(mime)}`, e, { fetch: doFetch, language: lang });+ return c.json({ text, engine: e.label });+ } catch (err) {+ const message = err instanceof Error ? err.message : String(err);+ return c.json({ error: message === 'nothing was recorded' ? "Didn't catch anything. Check the microphone and try again." : `Couldn't transcribe: ${message}`, engine: e.label }, 502);+ }+});++export default transcribe;diff --git a/src/web/server.ts b/src/web/server.tsindex 463aaa60..7548063a 100644--- a/src/web/server.ts+++ b/src/web/server.ts@@ -13,6 +13,7 @@ import authRoutes from './api/auth.js';import statusRoutes, { updateStatus } from './api/status.js';import providerRoutes from './api/providers.js';import cloudRoutes from './api/cloud.js';+import transcribeRoutes from './api/transcribe.js';import configRoutes from './api/config.js';import systemRoutes, { setScheduler } from './api/system.js';import brainRoutes, { setUserMemory } from './api/brain.js';@@ -93,6 +94,7 @@ app.route('/', authRoutes);app.route('/', statusRoutes);app.route('/', providerRoutes);app.route('/', cloudRoutes);+app.route('/', transcribeRoutes);app.route('/', configRoutes);app.route('/', systemRoutes);app.route('/', brainRoutes);