Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat: read replies aloud

Salman Qureshicff9cc7f1269
diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json
index de5dbc41..26512410 100644
--- a/apps/desktop/src-tauri/tauri.conf.json
+++ b/apps/desktop/src-tauri/tauri.conf.json
@@ -33,7 +33,7 @@
"style-src": "'self' 'unsafe-inline' https://fonts.googleapis.com",
"font-src": "'self' data: https://fonts.gstatic.com",
"img-src": "'self' data: blob: asset: http://asset.localhost http: https:",
- "media-src": "'self' blob: http: https:",
+ "media-src": "'self' blob: data: http: https:",
"connect-src": "'self' ipc: http://ipc.localhost http://127.0.0.1:* http://localhost:* http: https: ws: wss:",
"worker-src": "'self' blob:",
"frame-src": "'none'",
diff --git a/apps/desktop/src/api/models.ts b/apps/desktop/src/api/models.ts
index 2dc25d00..e9677e9b 100644
--- a/apps/desktop/src/api/models.ts
+++ b/apps/desktop/src/api/models.ts
@@ -1,4 +1,4 @@
-import { http, request } from "./client";
+import { ApiError, apiUrl, authHeaders, http, request } from "./client";
import type { PluginItem } from "./types";
/** Model providers and gateways (GET/POST /api/providers*), voice key checks, and Optimus voice settings. */
@@ -138,6 +138,24 @@ export const cloudApi = {
};
/** Dictation: GET tells whether a speech service is set up; POST sends raw audio and gets text back. */
+export const speakApi = {
+ status: () => http.get<{ available: boolean; engine: string | null; maxChars: number }>("/api/speak"),
+ /** MP3 for one part of a reply (at most `maxChars`). */
+ audio: async (text: string, signal?: AbortSignal): Promise<Blob> => {
+ const res = await fetch(apiUrl("/api/speak"), {
+ method: "POST",
+ headers: { "Content-Type": "application/json", Accept: "audio/mpeg, application/json", ...authHeaders() },
+ body: JSON.stringify({ text }),
+ signal,
+ });
+ if (!res.ok) {
+ const body = (await res.json().catch(() => ({}))) as { error?: string } & Record<string, unknown>;
+ throw new ApiError(body.error || `Request failed (${res.status})`, res.status, null, body);
+ }
+ return res.blob();
+ },
+};
+
export const transcribeApi = {
status: () => http.get<{ available: boolean; engine: string | null }>("/api/transcribe"),
send: (audio: Blob, language?: string) =>
diff --git a/apps/desktop/src/components/readAloud/ReadAloud.tsx b/apps/desktop/src/components/readAloud/ReadAloud.tsx
new file mode 100644
index 00000000..59af363d
--- /dev/null
+++ b/apps/desktop/src/components/readAloud/ReadAloud.tsx
@@ -0,0 +1,54 @@
+import { Loader2, Square, Volume2, X } from "lucide-react";
+import { useUi } from "~/app/store";
+import { Button } from "~/components/ui/Button";
+import { cx } from "~/lib/cx";
+import { useReadAloud } from "./speech";
+
+/** The speaker beside Copy: read this reply aloud, or stop. */
+export function ReadAloudButton({ id, text }: { id: string; text: string }) {
+ const mine = useReadAloud((s) => s.id === id);
+ const phase = useReadAloud((s) => (s.id === id ? s.phase : null));
+ const play = useReadAloud((s) => s.play);
+ const stop = useReadAloud((s) => s.stop);
+ const active = mine && (phase === "loading" || phase === "playing");
+ return (
+ <button
+ onClick={() => (active ? stop() : play(id, text))}
+ className={cx("flex h-6 items-center gap-1 rounded-md px-1.5 text-xs hover:bg-ink/[0.06] hover:text-ink", active ? "text-accent" : "text-ink4")}
+ aria-label={active ? "Stop reading aloud" : "Read aloud"}
+ aria-pressed={active}
+ >
+ {phase === "loading" ? <Loader2 size={13} className="animate-spin" /> : active ? <Square size={11} className="fill-current" /> : <Volume2 size={13} />}
+ {active ? "Stop" : "Listen"}
+ </button>
+ );
+}
+
+/** Is this reply being read (keeps its actions visible while it plays)? */
+export function useReadingThis(id: string): boolean {
+ return useReadAloud((s) => s.id === id && s.phase !== null);
+}
+
+/** No speech service, or a failure: one line and a way forward, under the reply. */
+export function ReadAloudNotice({ id }: { id: string }) {
+ const phase = useReadAloud((s) => (s.id === id ? s.phase : null));
+ const error = useReadAloud((s) => s.error);
+ const dismiss = useReadAloud((s) => s.dismiss);
+ const openSettings = useUi((s) => s.openSettings);
+ if (phase !== "needs-setup" && phase !== "error") return null;
+ return (
+ <div className="mt-2 flex flex-wrap items-center gap-2 rounded-lg border border-amber/40 bg-amber/[0.06] px-3 py-2 text-xs text-ink2" role="alert">
+ <Volume2 size={13} className="shrink-0 text-amber" />
+ <span className="min-w-0 flex-1">
+ {phase === "needs-setup" ? "Reading aloud needs a speech service. Add an OpenAI key under Models, or a Deepgram or ElevenLabs key under Voice." : error}
+ </span>
+ {phase === "needs-setup" && (
+ <>
+ <Button type="button" size="sm" variant="primary" onClick={() => { dismiss(); openSettings("models"); }}>Add OpenAI key</Button>
+ <Button type="button" size="sm" variant="outline" onClick={() => { dismiss(); openSettings("voice"); }}>Voice settings</Button>
+ </>
+ )}
+ <button type="button" onClick={dismiss} className="text-ink4 hover:text-ink" aria-label="Dismiss"><X size={13} /></button>
+ </div>
+ );
+}
diff --git a/apps/desktop/src/components/readAloud/speech.test.ts b/apps/desktop/src/components/readAloud/speech.test.ts
new file mode 100644
index 00000000..1481ed8d
--- /dev/null
+++ b/apps/desktop/src/components/readAloud/speech.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+import { speakableText, speechParts } from "./speech";
+
+describe("read aloud text", () => {
+ it("reads prose, not markdown, code or link addresses", () => {
+ const md = [
+ "## Weather today",
+ "",
+ "It's **sunny** and *warm*. See [the forecast](https://example.com/f) or https://x.y/z.",
+ "",
+ "- Take water",
+ "1. Leave at 9",
+ "",
+ "```bash",
+ "curl wttr.in",
+ "```",
+ "",
+ "Use `npm run dev` to start.",
+ ].join("\n");
+ const out = speakableText(md);
+ expect(out).toContain("Weather today.");
+ expect(out).toContain("It's sunny and warm. See the forecast or a link.");
+ expect(out).toContain("Take water");
+ expect(out).toContain("Leave at 9");
+ expect(out).toContain("(code omitted)");
+ expect(out).toContain("Use npm run dev to start.");
+ expect(out).not.toMatch(/[#*`]|https?:|curl/);
+ });
+
+ it("reads a table row by row", () => {
+ expect(speakableText("| City | Temp |\n|---|---|\n| Srinagar | 12 |")).toBe("City, Temp.\nSrinagar, 12.");
+ });
+
+ it("splits into parts at sentence boundaries, a short one first", () => {
+ const sentences = Array.from({ length: 40 }, (_, i) => `Sentence number ${i} is here.`);
+ const parts = speechParts(sentences.join(" "), 100, 400);
+ expect(parts[0].length).toBeLessThanOrEqual(100);
+ expect(parts.every((p) => p.length <= 400)).toBe(true);
+ expect(parts.join(" ")).toBe(sentences.join(" "));
+ expect(parts.every((p) => p.endsWith("."))).toBe(true);
+ });
+
+ it("cuts a single huge sentence at word boundaries", () => {
+ const long = Array.from({ length: 300 }, () => "word").join(" ");
+ const parts = speechParts(long, 100, 400);
+ expect(parts.every((p) => p.length <= 400 && !p.startsWith(" "))).toBe(true);
+ expect(parts.join(" ")).toBe(long);
+ });
+});
diff --git a/apps/desktop/src/components/readAloud/speech.ts b/apps/desktop/src/components/readAloud/speech.ts
new file mode 100644
index 00000000..74f63c5a
--- /dev/null
+++ b/apps/desktop/src/components/readAloud/speech.ts
@@ -0,0 +1,141 @@
+import { create } from "zustand";
+import { ApiError } from "~/api/client";
+import { speakApi } from "~/api/models";
+
+/**
+ * Read a reply aloud with Mercury's speech service. One reply at a time
+ * across the app: starting another stops the first.
+ *
+ * The reply is turned into plain speakable text (code, tables and link
+ * addresses aren't read out), split into parts at sentence boundaries, and
+ * each part is fetched while the one before plays, so the first words come
+ * quickly even for a long answer.
+ */
+
+export type ReadPhase = "loading" | "playing" | "needs-setup" | "error";
+
+interface ReadAloudState {
+ /** The message being read (or that needs attention), and how it's going. */
+ id: string | null;
+ phase: ReadPhase | null;
+ error: string | null;
+ play(id: string, markdown: string): void;
+ stop(): void;
+ dismiss(): void;
+}
+
+/** Each part stays well under the runtime's 4000-character limit; small first parts start sooner. */
+const FIRST_PART = 280;
+const PART = 1500;
+
+/** Markdown → what a person would say out loud. */
+export function speakableText(markdown: string): string {
+ return markdown
+ .replace(/```[\s\S]*?(```|$)/g, "\n(code omitted)\n")
+ .replace(/^[ \t]*\|?[ \t]*:?-{3,}[-|: \t]*\n?/gm, "")
+ .replace(/^[ \t]*\|.*\|[ \t]*$/gm, (row) => row.split("|").map((c) => c.trim()).filter(Boolean).join(", ") + ".")
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
+ .replace(/<?https?:\/\/[^\s>)]*[^\s>).,;:!?'"]>?/g, "a link")
+ .replace(/`([^`]+)`/g, "$1")
+ .replace(/^[ \t]{0,3}#{1,6}[ \t]+(.*)$/gm, "$1.")
+ .replace(/^[ \t]*>[ \t]?/gm, "")
+ .replace(/^[ \t]*[-*+][ \t]+\[[ xX]\][ \t]+/gm, "")
+ .replace(/^[ \t]*(?:[-*+]|\d+[.)])[ \t]+/gm, "")
+ .replace(/(\*\*|__)(.+?)\1/g, "$2")
+ .replace(/(^|[^\w*])[*_]([^*_\n]+)[*_](?=[^\w*]|$)/g, "$1$2")
+ .replace(/~~(.+?)~~/g, "$1")
+ .replace(/^[ \t]*(?:---|\*\*\*|___)[ \t]*$/gm, "")
+ .replace(/<\/?[a-z][^>]*>/gi, "")
+ .replace(/[ \t]+/g, " ")
+ .replace(/\.(\s*\.)+/g, ".")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+}
+
+/** Split into parts at paragraph, then sentence, then word boundaries. */
+export function speechParts(text: string, first = FIRST_PART, size = PART): string[] {
+ const sentences = text.split(/(?<=[.!?…:;])\s+|\n+/).map((s) => s.trim()).filter(Boolean);
+ const parts: string[] = [];
+ let current = "";
+ const limit = () => (parts.length === 0 ? first : size);
+ const push = () => { if (current) parts.push(current); current = ""; };
+ for (let sentence of sentences) {
+ while (sentence.length > size) {
+ // One very long sentence: cut at a word boundary.
+ push();
+ const cut = sentence.lastIndexOf(" ", size) > size / 2 ? sentence.lastIndexOf(" ", size) : size;
+ parts.push(sentence.slice(0, cut).trim());
+ sentence = sentence.slice(cut).trim();
+ }
+ if (current && current.length + 1 + sentence.length > limit()) push();
+ current = current ? `${current} ${sentence}` : sentence;
+ }
+ push();
+ return parts;
+}
+
+let session: { abort: AbortController; audio: HTMLAudioElement | null; urls: string[] } | null = null;
+
+function teardown() {
+ if (!session) return;
+ session.abort.abort();
+ session.audio?.pause();
+ session.urls.forEach((u) => URL.revokeObjectURL(u));
+ session = null;
+}
+
+function playBlob(blob: Blob, s: NonNullable<typeof session>): Promise<void> {
+ return new Promise((resolve, reject) => {
+ const url = URL.createObjectURL(blob);
+ s.urls.push(url);
+ const audio = new Audio(url);
+ s.audio = audio;
+ const done = () => { s.abort.signal.removeEventListener("abort", done); resolve(); };
+ audio.onended = done;
+ audio.onerror = () => reject(new Error("This audio couldn't be played."));
+ s.abort.signal.addEventListener("abort", done);
+ audio.play().catch(reject);
+ });
+}
+
+export const useReadAloud = create<ReadAloudState>((set) => ({
+ id: null,
+ phase: null,
+ error: null,
+ stop: () => { teardown(); set({ id: null, phase: null, error: null }); },
+ dismiss: () => set({ id: null, phase: null, error: null }),
+ play: (id, markdown) => {
+ teardown();
+ const parts = speechParts(speakableText(markdown));
+ if (!parts.length) return;
+ const s = { abort: new AbortController(), audio: null, urls: [] as string[] };
+ session = s;
+ const mine = () => session === s;
+ set({ id, phase: "loading", error: null });
+ void (async () => {
+ try {
+ let next: Promise<Blob> = speakApi.audio(parts[0], s.abort.signal);
+ for (let i = 0; i < parts.length; i++) {
+ const blob = await next;
+ if (!mine()) return;
+ // Fetch the next part while this one plays.
+ if (i + 1 < parts.length) {
+ next = speakApi.audio(parts[i + 1], s.abort.signal);
+ next.catch(() => {}); // handled when awaited
+ }
+ set({ phase: "playing" });
+ await playBlob(blob, s);
+ if (!mine()) return;
+ }
+ if (mine()) { teardown(); set({ id: null, phase: null }); }
+ } catch (err) {
+ if (!mine()) return;
+ teardown();
+ if (err instanceof ApiError && err.data.needs === "speech") set({ id, phase: "needs-setup", error: null });
+ else if (err instanceof ApiError && err.status === 404) set({ id, phase: "error", error: "This Mercury can't read aloud yet. Update Mercury to use it." });
+ else set({ id, phase: "error", error: err instanceof Error ? err.message : String(err) });
+ }
+ })();
+ },
+}));
diff --git a/apps/desktop/src/views/chat/Message.tsx b/apps/desktop/src/views/chat/Message.tsx
index 2434e5b4..ec4be956 100644
--- a/apps/desktop/src/views/chat/Message.tsx
+++ b/apps/desktop/src/views/chat/Message.tsx
@@ -12,6 +12,7 @@ import { isStopNote } from "~/app/events";
import { UserAvatar, useDisplayName } from "~/components/UserAvatar";
import { usePulse } from "~/app/pulse";
import { ArtifactStrip } from "~/components/artifacts/ArtifactCard";
+import { ReadAloudButton, ReadAloudNotice, useReadingThis } from "~/components/readAloud/ReadAloud";
/**
* One entry in a conversation. Each kind the runtime stores gets its own
@@ -69,7 +70,8 @@ export const Message = memo(function Message({ message }: { message: ChatMessage
{body && <Markdown text={body} />}
{stopped && <StoppedMarker partial={!!body} />}
{message.artifacts && message.artifacts.length > 0 && <ArtifactStrip items={message.artifacts} />}
- <MessageMeta timestamp={message.timestamp} copy={body || undefined} />
+ <MessageMeta timestamp={message.timestamp} copy={body || undefined} listen={body ? { id: message.id, text: body } : undefined} />
+ <ReadAloudNotice id={message.id} />
</div>
</div>
);
@@ -97,11 +99,12 @@ function Speaker({ who }: { who: "you" | "mercury" }) {
}
/** Time and copy, shown on hover so the conversation stays quiet. */
-function MessageMeta({ timestamp, copy, align = "start" }: { timestamp: string; copy?: string; align?: "start" | "end" }) {
+function MessageMeta({ timestamp, copy, listen, align = "start" }: { timestamp: string; copy?: string; listen?: { id: string; text: string }; align?: "start" | "end" }) {
const [copied, setCopied] = useState(false);
+ const reading = useReadingThis(listen?.id ?? "");
const time = formatTime(timestamp);
return (
- <div className={cx("-mb-2 flex h-6 items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100 focus-within:opacity-100", align === "end" ? "mt-0 flex-row-reverse" : "mt-2")}>
+ <div className={cx("-mb-2 flex h-6 items-center gap-1 transition-opacity group-hover:opacity-100 focus-within:opacity-100", reading ? "opacity-100" : "opacity-0", align === "end" ? "mt-0 flex-row-reverse" : "mt-2")}>
{copy && (
<button
onClick={() => { void navigator.clipboard.writeText(copy).then(() => { setCopied(true); setTimeout(() => setCopied(false), 1200); }); }}
@@ -112,6 +115,7 @@ function MessageMeta({ timestamp, copy, align = "start" }: { timestamp: string;
{copied ? "Copied" : "Copy"}
</button>
)}
+ {listen && <ReadAloudButton id={listen.id} text={listen.text} />}
{time && <span className="px-1.5 text-xs text-ink4">{time}</span>}
</div>
);
diff --git a/src/plugins/speak.ts b/src/plugins/speak.ts
new file mode 100644
index 00000000..7d0e663c
--- /dev/null
+++ b/src/plugins/speak.ts
@@ -0,0 +1,82 @@
+/**
+ * Text to speech, with the speech service the owner already set up (the
+ * same one voice typing uses, see pickSpeechEngine): OpenAI
+ * (gpt-4o-mini-tts, falling back to tts-1), Deepgram (Aura 2) or
+ * ElevenLabs (Flash v2.5). Always MP3, so every client can play it.
+ * Used to read Mercury's replies aloud in the apps.
+ */
+import type { SpeechEngine } from './transcribe.js';
+
+/** Longest text spoken in one request; clients send a long reply in parts. */
+export const SPEAK_MAX_CHARS = 4000;
+/** Deepgram's own limit per request. */
+const DEEPGRAM_MAX_CHARS = 2000;
+
+const OPENAI_VOICE = 'coral';
+const DEEPGRAM_MODEL = 'aura-2-thalia-en';
+const ELEVENLABS_VOICE = 'JBFqnCBsd6RMkjVDRZzb';
+const ELEVENLABS_MODEL = 'eleven_flash_v2_5';
+
+type Fetch = typeof fetch;
+
+export interface SpeakOptions {
+ fetch?: Fetch;
+ signal?: AbortSignal;
+}
+
+/** Speak `text` with `engine`; resolves with MP3 bytes. */
+export async function synthesizeSpeech(text: string, engine: SpeechEngine, opts: SpeakOptions = {}): Promise<Uint8Array> {
+ const input = text.trim();
+ if (!input) throw new Error('nothing to say');
+ const f = opts.fetch ?? fetch;
+ const signal = opts.signal;
+
+ if (engine.kind === 'deepgram') {
+ const res = await f(`https://api.deepgram.com/v1/speak?model=${DEEPGRAM_MODEL}&encoding=mp3`, {
+ method: 'POST',
+ headers: { Authorization: `Token ${engine.apiKey}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ text: input.slice(0, DEEPGRAM_MAX_CHARS) }),
+ signal,
+ });
+ return audioOf(res, 'Deepgram');
+ }
+
+ if (engine.kind === 'elevenlabs') {
+ const res = await f(`https://api.elevenlabs.io/v1/text-to-speech/${ELEVENLABS_VOICE}?output_format=mp3_44100_128`, {
+ method: 'POST',
+ headers: { 'xi-api-key': engine.apiKey, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ text: input.slice(0, SPEAK_MAX_CHARS), model_id: ELEVENLABS_MODEL }),
+ signal,
+ });
+ return audioOf(res, 'ElevenLabs');
+ }
+
+ const base = (engine.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');
+ const say = (model: string) => f(`${base}/audio/speech`, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${engine.apiKey}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ model, voice: OPENAI_VOICE, input: input.slice(0, SPEAK_MAX_CHARS), response_format: 'mp3' }),
+ signal,
+ });
+ let res = await say('gpt-4o-mini-tts');
+ // An OpenAI-compatible endpoint without the newer model: try the classic one.
+ if (res.status === 400 || res.status === 404) res = await say('tts-1');
+ return audioOf(res, 'OpenAI');
+}
+
+async function audioOf(res: Response, vendor: string): Promise<Uint8Array> {
+ if (!res.ok) {
+ let detail = '';
+ try {
+ const body = (await res.json()) as { error?: { message?: string } | string; detail?: { message?: string } | string; err_msg?: string };
+ const e = body.error ?? body.detail ?? body.err_msg;
+ detail = typeof e === 'string' ? e : e?.message ?? '';
+ } catch { /* not JSON */ }
+ if (res.status === 401 || res.status === 403) throw new Error(`${vendor} rejected the key. Check it in Settings.`);
+ if (res.status === 429) throw new Error(`${vendor} is rate-limiting right now. Try again in a moment.`);
+ throw new Error(detail ? `${vendor}: ${detail.slice(0, 200)}` : `${vendor} answered ${res.status}`);
+ }
+ const bytes = new Uint8Array(await res.arrayBuffer());
+ if (bytes.byteLength === 0) throw new Error(`${vendor} sent no audio`);
+ return bytes;
+}
diff --git a/src/web/api-contract.ts b/src/web/api-contract.ts
index 05202ad0..9129a98b 100644
--- a/src/web/api-contract.ts
+++ b/src/web/api-contract.ts
@@ -45,6 +45,7 @@ export const API_FEATURES: readonly string[] = Object.freeze([
'chat.plan', // SSE 'plan' { steps, updatedAt } + GET /api/chat/plan?sessionId= (the plan checklist)
'channels', // GET /api/channels, POST :type/check, PUT :type, POST :type/access, POST/GET signal/link
'transcribe', // GET/POST /api/transcribe — dictation: raw audio in, text out
+ 'speak', // GET/POST /api/speak — read aloud: text in, MP3 out
'cloud.signin', // GET /api/cloud/status, POST/GET/DELETE /api/cloud/signin (browser pairing from any client) // POST /api/voice/check { vendor, key?, endpoint? }
'brain.map', // GET /api/brain/map[?archive=1] — themes, sub-themes, notes, related links
]);
diff --git a/src/web/api/speak-route.test.ts b/src/web/api/speak-route.test.ts
new file mode 100644
index 00000000..6a6005d4
--- /dev/null
+++ b/src/web/api/speak-route.test.ts
@@ -0,0 +1,79 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+const home = await vi.hoisted(async () => {
+ const fs = await import('node:fs');
+ const os = await import('node:os');
+ const path = await import('node:path');
+ for (const k of ['OPENAI_API_KEY', 'DEEPGRAM_API_KEY', 'ELEVENLABS_API_KEY']) delete process.env[k];
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-speak-'));
+ fs.writeFileSync(path.join(dir, 'mercury.yaml'), 'identity:\n owner: Test\n');
+ process.env.MERCURY_HOME = dir;
+ return dir;
+});
+
+import speak, { setSpeakFetch } from './speak.js';
+
+const mp3 = new Uint8Array([0xff, 0xfb, 0x90, 0x44, 1, 2, 3]);
+const audioReply = () => ({ ok: true, status: 200, arrayBuffer: async () => mp3.buffer.slice(0) });
+const post = (text: unknown) => speak.request('http://127.0.0.1/api/speak', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text }) });
+const withOpenAI = () => writeFileSync(join(home, 'mercury.yaml'), [
+ 'identity:', ' owner: Test', 'providers:', ' openai:', ' name: openai', ' apiKey: sk-test-openai-000000000000', ' baseUrl: https://api.openai.com/v1', ' model: gpt-4o-mini', ' enabled: true', '',
+].join('\n'));
+
+afterEach(() => { setSpeakFetch(undefined); delete process.env.DEEPGRAM_API_KEY; delete process.env.ELEVENLABS_API_KEY; });
+
+describe('read aloud', () => {
+ it('with no speech service, says how to turn one on', async () => {
+ expect(await (await speak.request('http://127.0.0.1/api/speak')).json()).toEqual({ available: false, engine: null, maxChars: 4000 });
+ const res = await post('Hello');
+ expect(res.status).toBe(412);
+ expect(await res.json()).toMatchObject({ needs: 'speech' });
+ });
+
+ it('speaks with the OpenAI provider and returns MP3', async () => {
+ withOpenAI();
+ const f = vi.fn().mockResolvedValue(audioReply());
+ setSpeakFetch(f as never);
+ const res = await post('The weather is sunny.');
+ expect(res.status).toBe(200);
+ expect(res.headers.get('content-type')).toBe('audio/mpeg');
+ expect(new Uint8Array(await res.arrayBuffer())).toEqual(mp3);
+ expect(f.mock.calls[0][0]).toBe('https://api.openai.com/v1/audio/speech');
+ expect(JSON.parse(f.mock.calls[0][1].body)).toMatchObject({ model: 'gpt-4o-mini-tts', input: 'The weather is sunny.', response_format: 'mp3' });
+ });
+
+ it('falls back to tts-1 on an endpoint without the newer model', async () => {
+ withOpenAI();
+ const f = vi.fn().mockResolvedValueOnce({ ok: false, status: 404, json: async () => ({}) }).mockResolvedValueOnce(audioReply());
+ setSpeakFetch(f as never);
+ expect((await post('Hi')).status).toBe(200);
+ expect(JSON.parse(f.mock.calls[1][1].body).model).toBe('tts-1');
+ });
+
+ it('uses a Deepgram or ElevenLabs voice key when there is no OpenAI', async () => {
+ writeFileSync(join(home, 'mercury.yaml'), 'identity:\n owner: Test\n');
+ process.env.ELEVENLABS_API_KEY = 'el-key';
+ const f = vi.fn().mockResolvedValue(audioReply());
+ setSpeakFetch(f as never);
+ expect((await post('Hi there')).status).toBe(200);
+ expect(f.mock.calls[0][0]).toContain('api.elevenlabs.io/v1/text-to-speech/');
+ expect(f.mock.calls[0][1].headers['xi-api-key']).toBe('el-key');
+
+ process.env.DEEPGRAM_API_KEY = 'dg-key';
+ expect((await post('Hi there')).status).toBe(200);
+ expect(f.mock.calls[1][0]).toContain('api.deepgram.com/v1/speak');
+ expect(f.mock.calls[1][1].headers.Authorization).toBe('Token dg-key');
+ });
+
+ it('refuses empty or oversized text, and a rejected key reads plainly', async () => {
+ withOpenAI();
+ expect((await post(' ')).status).toBe(400);
+ expect((await post('x'.repeat(4001))).status).toBe(413);
+ setSpeakFetch(vi.fn().mockResolvedValue({ ok: false, status: 401, json: async () => ({ error: { message: 'bad key' } }) }) as never);
+ const res = await post('Hello');
+ expect(res.status).toBe(502);
+ expect((await res.json() as any).error).toMatch(/rejected the key/);
+ });
+});
diff --git a/src/web/api/speak.ts b/src/web/api/speak.ts
new file mode 100644
index 00000000..26a37138
--- /dev/null
+++ b/src/web/api/speak.ts
@@ -0,0 +1,53 @@
+import { Hono } from 'hono';
+import { loadConfig } from '../../utils/config.js';
+import { secretValue } from '../../plugins/store.js';
+import { pickSpeechEngine } from '../../plugins/transcribe.js';
+import { SPEAK_MAX_CHARS, synthesizeSpeech } from '../../plugins/speak.js';
+
+/**
+ * Read aloud for every client: text in, speech out.
+ *
+ * GET /api/speak { available, engine, maxChars } — which speech service would be used
+ * POST /api/speak { text } → audio/mpeg
+ *
+ * Uses the speech service voice typing uses (OpenAI provider, else a
+ * Deepgram or ElevenLabs voice key). With none, 412 { needs: 'speech' } so
+ * the app can say how to turn it on. Clients send a long reply in parts of
+ * at most `maxChars`, so the first words play while the rest is made.
+ */
+
+type Fetch = typeof fetch;
+let doFetch: Fetch | undefined;
+/** Test seam. */
+export function setSpeakFetch(f: Fetch | undefined): void { doFetch = f; }
+
+function engine() {
+ return pickSpeechEngine(loadConfig().providers.openai, secretValue);
+}
+
+const speak = new Hono();
+
+speak.get('/api/speak', (c) => {
+ const e = engine();
+ return c.json({ available: !!e, engine: e?.label ?? null, maxChars: SPEAK_MAX_CHARS });
+});
+
+speak.post('/api/speak', async (c) => {
+ const e = engine();
+ if (!e) {
+ return c.json({ error: 'Reading aloud needs a speech service: an OpenAI key (Settings → Models), or a Deepgram or ElevenLabs key (Settings → Voice).', needs: 'speech' }, 412);
+ }
+ const body = (await c.req.json().catch(() => ({}))) as { text?: unknown };
+ const text = typeof body.text === 'string' ? body.text.trim() : '';
+ if (!text) return c.json({ error: 'Nothing to read.' }, 400);
+ if (text.length > SPEAK_MAX_CHARS) return c.json({ error: `Send at most ${SPEAK_MAX_CHARS} characters at a time.`, maxChars: SPEAK_MAX_CHARS }, 413);
+ try {
+ const audio = await synthesizeSpeech(text, e, { fetch: doFetch, signal: c.req.raw.signal });
+ return new Response(audio, { headers: { 'Content-Type': 'audio/mpeg', 'Cache-Control': 'no-store', 'X-Speech-Engine': e.label } });
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ return c.json({ error: `Couldn't read that aloud: ${message}`, engine: e.label }, 502);
+ }
+});
+
+export default speak;
diff --git a/src/web/server.ts b/src/web/server.ts
index d64d157a..e47ad9dd 100644
--- a/src/web/server.ts
+++ b/src/web/server.ts
@@ -14,6 +14,7 @@ import statusRoutes, { updateStatus } from './api/status.js';
import providerRoutes from './api/providers.js';
import cloudRoutes from './api/cloud.js';
import transcribeRoutes from './api/transcribe.js';
+import speakRoutes from './api/speak.js';
import channelRoutes from './api/channels.js';
import artifactRoutes from './api/artifacts.js';
import configRoutes from './api/config.js';
@@ -97,6 +98,7 @@ app.route('/', statusRoutes);
app.route('/', providerRoutes);
app.route('/', cloudRoutes);
app.route('/', transcribeRoutes);
+app.route('/', speakRoutes);
app.route('/', channelRoutes);
app.route('/', artifactRoutes);
app.route('/', configRoutes);