Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat: read replies aloud
Salman Qureshicff9cc7f1269
diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.jsonindex de5dbc41..26512410 100644--- a/apps/desktop/src-tauri/tauri.conf.json+++ b/apps/desktop/src-tauri/tauri.conf.json@@ -33,7 +33,7 @@"style-src": "'self' 'unsafe-inline' https://fonts.googleapis.com","font-src": "'self' data: https://fonts.gstatic.com","img-src": "'self' data: blob: asset: http://asset.localhost http: https:",- "media-src": "'self' blob: http: https:",+ "media-src": "'self' blob: data: http: https:","connect-src": "'self' ipc: http://ipc.localhost http://127.0.0.1:* http://localhost:* http: https: ws: wss:","worker-src": "'self' blob:","frame-src": "'none'",diff --git a/apps/desktop/src/api/models.ts b/apps/desktop/src/api/models.tsindex 2dc25d00..e9677e9b 100644--- a/apps/desktop/src/api/models.ts+++ b/apps/desktop/src/api/models.ts@@ -1,4 +1,4 @@-import { http, request } from "./client";+import { ApiError, apiUrl, authHeaders, http, request } from "./client";import type { PluginItem } from "./types";/** Model providers and gateways (GET/POST /api/providers*), voice key checks, and Optimus voice settings. */@@ -138,6 +138,24 @@ export const cloudApi = {};/** Dictation: GET tells whether a speech service is set up; POST sends raw audio and gets text back. */+export const speakApi = {+ status: () => http.get<{ available: boolean; engine: string | null; maxChars: number }>("/api/speak"),+ /** MP3 for one part of a reply (at most `maxChars`). */+ audio: async (text: string, signal?: AbortSignal): Promise<Blob> => {+ const res = await fetch(apiUrl("/api/speak"), {+ method: "POST",+ headers: { "Content-Type": "application/json", Accept: "audio/mpeg, application/json", ...authHeaders() },+ body: JSON.stringify({ text }),+ signal,+ });+ if (!res.ok) {+ const body = (await res.json().catch(() => ({}))) as { error?: string } & Record<string, unknown>;+ throw new ApiError(body.error || `Request failed (${res.status})`, res.status, null, body);+ }+ return res.blob();+ },+};+export const transcribeApi = {status: () => http.get<{ available: boolean; engine: string | null }>("/api/transcribe"),send: (audio: Blob, language?: string) =>diff --git a/apps/desktop/src/components/readAloud/ReadAloud.tsx b/apps/desktop/src/components/readAloud/ReadAloud.tsxnew file mode 100644index 00000000..59af363d--- /dev/null+++ b/apps/desktop/src/components/readAloud/ReadAloud.tsx@@ -0,0 +1,54 @@+import { Loader2, Square, Volume2, X } from "lucide-react";+import { useUi } from "~/app/store";+import { Button } from "~/components/ui/Button";+import { cx } from "~/lib/cx";+import { useReadAloud } from "./speech";++/** The speaker beside Copy: read this reply aloud, or stop. */+export function ReadAloudButton({ id, text }: { id: string; text: string }) {+ const mine = useReadAloud((s) => s.id === id);+ const phase = useReadAloud((s) => (s.id === id ? s.phase : null));+ const play = useReadAloud((s) => s.play);+ const stop = useReadAloud((s) => s.stop);+ const active = mine && (phase === "loading" || phase === "playing");+ return (+ <button+ onClick={() => (active ? stop() : play(id, text))}+ className={cx("flex h-6 items-center gap-1 rounded-md px-1.5 text-xs hover:bg-ink/[0.06] hover:text-ink", active ? "text-accent" : "text-ink4")}+ aria-label={active ? "Stop reading aloud" : "Read aloud"}+ aria-pressed={active}+ >+ {phase === "loading" ? <Loader2 size={13} className="animate-spin" /> : active ? <Square size={11} className="fill-current" /> : <Volume2 size={13} />}+ {active ? "Stop" : "Listen"}+ </button>+ );+}++/** Is this reply being read (keeps its actions visible while it plays)? */+export function useReadingThis(id: string): boolean {+ return useReadAloud((s) => s.id === id && s.phase !== null);+}++/** No speech service, or a failure: one line and a way forward, under the reply. */+export function ReadAloudNotice({ id }: { id: string }) {+ const phase = useReadAloud((s) => (s.id === id ? s.phase : null));+ const error = useReadAloud((s) => s.error);+ const dismiss = useReadAloud((s) => s.dismiss);+ const openSettings = useUi((s) => s.openSettings);+ if (phase !== "needs-setup" && phase !== "error") return null;+ return (+ <div className="mt-2 flex flex-wrap items-center gap-2 rounded-lg border border-amber/40 bg-amber/[0.06] px-3 py-2 text-xs text-ink2" role="alert">+ <Volume2 size={13} className="shrink-0 text-amber" />+ <span className="min-w-0 flex-1">+ {phase === "needs-setup" ? "Reading aloud needs a speech service. Add an OpenAI key under Models, or a Deepgram or ElevenLabs key under Voice." : error}+ </span>+ {phase === "needs-setup" && (+ <>+ <Button type="button" size="sm" variant="primary" onClick={() => { dismiss(); openSettings("models"); }}>Add OpenAI key</Button>+ <Button type="button" size="sm" variant="outline" onClick={() => { dismiss(); openSettings("voice"); }}>Voice settings</Button>+ </>+ )}+ <button type="button" onClick={dismiss} className="text-ink4 hover:text-ink" aria-label="Dismiss"><X size={13} /></button>+ </div>+ );+}diff --git a/apps/desktop/src/components/readAloud/speech.test.ts b/apps/desktop/src/components/readAloud/speech.test.tsnew file mode 100644index 00000000..1481ed8d--- /dev/null+++ b/apps/desktop/src/components/readAloud/speech.test.ts@@ -0,0 +1,49 @@+import { describe, expect, it } from "vitest";+import { speakableText, speechParts } from "./speech";++describe("read aloud text", () => {+ it("reads prose, not markdown, code or link addresses", () => {+ const md = [+ "## Weather today",+ "",+ "It's **sunny** and *warm*. See [the forecast](https://example.com/f) or https://x.y/z.",+ "",+ "- Take water",+ "1. Leave at 9",+ "",+ "```bash",+ "curl wttr.in",+ "```",+ "",+ "Use `npm run dev` to start.",+ ].join("\n");+ const out = speakableText(md);+ expect(out).toContain("Weather today.");+ expect(out).toContain("It's sunny and warm. See the forecast or a link.");+ expect(out).toContain("Take water");+ expect(out).toContain("Leave at 9");+ expect(out).toContain("(code omitted)");+ expect(out).toContain("Use npm run dev to start.");+ expect(out).not.toMatch(/[#*`]|https?:|curl/);+ });++ it("reads a table row by row", () => {+ expect(speakableText("| City | Temp |\n|---|---|\n| Srinagar | 12 |")).toBe("City, Temp.\nSrinagar, 12.");+ });++ it("splits into parts at sentence boundaries, a short one first", () => {+ const sentences = Array.from({ length: 40 }, (_, i) => `Sentence number ${i} is here.`);+ const parts = speechParts(sentences.join(" "), 100, 400);+ expect(parts[0].length).toBeLessThanOrEqual(100);+ expect(parts.every((p) => p.length <= 400)).toBe(true);+ expect(parts.join(" ")).toBe(sentences.join(" "));+ expect(parts.every((p) => p.endsWith("."))).toBe(true);+ });++ it("cuts a single huge sentence at word boundaries", () => {+ const long = Array.from({ length: 300 }, () => "word").join(" ");+ const parts = speechParts(long, 100, 400);+ expect(parts.every((p) => p.length <= 400 && !p.startsWith(" "))).toBe(true);+ expect(parts.join(" ")).toBe(long);+ });+});diff --git a/apps/desktop/src/components/readAloud/speech.ts b/apps/desktop/src/components/readAloud/speech.tsnew file mode 100644index 00000000..74f63c5a--- /dev/null+++ b/apps/desktop/src/components/readAloud/speech.ts@@ -0,0 +1,141 @@+import { create } from "zustand";+import { ApiError } from "~/api/client";+import { speakApi } from "~/api/models";++/**+ * Read a reply aloud with Mercury's speech service. One reply at a time+ * across the app: starting another stops the first.+ *+ * The reply is turned into plain speakable text (code, tables and link+ * addresses aren't read out), split into parts at sentence boundaries, and+ * each part is fetched while the one before plays, so the first words come+ * quickly even for a long answer.+ */++export type ReadPhase = "loading" | "playing" | "needs-setup" | "error";++interface ReadAloudState {+ /** The message being read (or that needs attention), and how it's going. */+ id: string | null;+ phase: ReadPhase | null;+ error: string | null;+ play(id: string, markdown: string): void;+ stop(): void;+ dismiss(): void;+}++/** Each part stays well under the runtime's 4000-character limit; small first parts start sooner. */+const FIRST_PART = 280;+const PART = 1500;++/** Markdown → what a person would say out loud. */+export function speakableText(markdown: string): string {+ return markdown+ .replace(/```[\s\S]*?(```|$)/g, "\n(code omitted)\n")+ .replace(/^[ \t]*\|?[ \t]*:?-{3,}[-|: \t]*\n?/gm, "")+ .replace(/^[ \t]*\|.*\|[ \t]*$/gm, (row) => row.split("|").map((c) => c.trim()).filter(Boolean).join(", ") + ".")+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")+ .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")+ .replace(/<?https?:\/\/[^\s>)]*[^\s>).,;:!?'"]>?/g, "a link")+ .replace(/`([^`]+)`/g, "$1")+ .replace(/^[ \t]{0,3}#{1,6}[ \t]+(.*)$/gm, "$1.")+ .replace(/^[ \t]*>[ \t]?/gm, "")+ .replace(/^[ \t]*[-*+][ \t]+\[[ xX]\][ \t]+/gm, "")+ .replace(/^[ \t]*(?:[-*+]|\d+[.)])[ \t]+/gm, "")+ .replace(/(\*\*|__)(.+?)\1/g, "$2")+ .replace(/(^|[^\w*])[*_]([^*_\n]+)[*_](?=[^\w*]|$)/g, "$1$2")+ .replace(/~~(.+?)~~/g, "$1")+ .replace(/^[ \t]*(?:---|\*\*\*|___)[ \t]*$/gm, "")+ .replace(/<\/?[a-z][^>]*>/gi, "")+ .replace(/[ \t]+/g, " ")+ .replace(/\.(\s*\.)+/g, ".")+ .replace(/\n{3,}/g, "\n\n")+ .trim();+}++/** Split into parts at paragraph, then sentence, then word boundaries. */+export function speechParts(text: string, first = FIRST_PART, size = PART): string[] {+ const sentences = text.split(/(?<=[.!?…:;])\s+|\n+/).map((s) => s.trim()).filter(Boolean);+ const parts: string[] = [];+ let current = "";+ const limit = () => (parts.length === 0 ? first : size);+ const push = () => { if (current) parts.push(current); current = ""; };+ for (let sentence of sentences) {+ while (sentence.length > size) {+ // One very long sentence: cut at a word boundary.+ push();+ const cut = sentence.lastIndexOf(" ", size) > size / 2 ? sentence.lastIndexOf(" ", size) : size;+ parts.push(sentence.slice(0, cut).trim());+ sentence = sentence.slice(cut).trim();+ }+ if (current && current.length + 1 + sentence.length > limit()) push();+ current = current ? `${current} ${sentence}` : sentence;+ }+ push();+ return parts;+}++let session: { abort: AbortController; audio: HTMLAudioElement | null; urls: string[] } | null = null;++function teardown() {+ if (!session) return;+ session.abort.abort();+ session.audio?.pause();+ session.urls.forEach((u) => URL.revokeObjectURL(u));+ session = null;+}++function playBlob(blob: Blob, s: NonNullable<typeof session>): Promise<void> {+ return new Promise((resolve, reject) => {+ const url = URL.createObjectURL(blob);+ s.urls.push(url);+ const audio = new Audio(url);+ s.audio = audio;+ const done = () => { s.abort.signal.removeEventListener("abort", done); resolve(); };+ audio.onended = done;+ audio.onerror = () => reject(new Error("This audio couldn't be played."));+ s.abort.signal.addEventListener("abort", done);+ audio.play().catch(reject);+ });+}++export const useReadAloud = create<ReadAloudState>((set) => ({+ id: null,+ phase: null,+ error: null,+ stop: () => { teardown(); set({ id: null, phase: null, error: null }); },+ dismiss: () => set({ id: null, phase: null, error: null }),+ play: (id, markdown) => {+ teardown();+ const parts = speechParts(speakableText(markdown));+ if (!parts.length) return;+ const s = { abort: new AbortController(), audio: null, urls: [] as string[] };+ session = s;+ const mine = () => session === s;+ set({ id, phase: "loading", error: null });+ void (async () => {+ try {+ let next: Promise<Blob> = speakApi.audio(parts[0], s.abort.signal);+ for (let i = 0; i < parts.length; i++) {+ const blob = await next;+ if (!mine()) return;+ // Fetch the next part while this one plays.+ if (i + 1 < parts.length) {+ next = speakApi.audio(parts[i + 1], s.abort.signal);+ next.catch(() => {}); // handled when awaited+ }+ set({ phase: "playing" });+ await playBlob(blob, s);+ if (!mine()) return;+ }+ if (mine()) { teardown(); set({ id: null, phase: null }); }+ } catch (err) {+ if (!mine()) return;+ teardown();+ if (err instanceof ApiError && err.data.needs === "speech") set({ id, phase: "needs-setup", error: null });+ else if (err instanceof ApiError && err.status === 404) set({ id, phase: "error", error: "This Mercury can't read aloud yet. Update Mercury to use it." });+ else set({ id, phase: "error", error: err instanceof Error ? err.message : String(err) });+ }+ })();+ },+}));diff --git a/apps/desktop/src/views/chat/Message.tsx b/apps/desktop/src/views/chat/Message.tsxindex 2434e5b4..ec4be956 100644--- a/apps/desktop/src/views/chat/Message.tsx+++ b/apps/desktop/src/views/chat/Message.tsx@@ -12,6 +12,7 @@ import { isStopNote } from "~/app/events";import { UserAvatar, useDisplayName } from "~/components/UserAvatar";import { usePulse } from "~/app/pulse";import { ArtifactStrip } from "~/components/artifacts/ArtifactCard";+import { ReadAloudButton, ReadAloudNotice, useReadingThis } from "~/components/readAloud/ReadAloud";/*** One entry in a conversation. Each kind the runtime stores gets its own@@ -69,7 +70,8 @@ export const Message = memo(function Message({ message }: { message: ChatMessage{body && <Markdown text={body} />}{stopped && <StoppedMarker partial={!!body} />}{message.artifacts && message.artifacts.length > 0 && <ArtifactStrip items={message.artifacts} />}- <MessageMeta timestamp={message.timestamp} copy={body || undefined} />+ <MessageMeta timestamp={message.timestamp} copy={body || undefined} listen={body ? { id: message.id, text: body } : undefined} />+ <ReadAloudNotice id={message.id} /></div></div>);@@ -97,11 +99,12 @@ function Speaker({ who }: { who: "you" | "mercury" }) {}/** Time and copy, shown on hover so the conversation stays quiet. */-function MessageMeta({ timestamp, copy, align = "start" }: { timestamp: string; copy?: string; align?: "start" | "end" }) {+function MessageMeta({ timestamp, copy, listen, align = "start" }: { timestamp: string; copy?: string; listen?: { id: string; text: string }; align?: "start" | "end" }) {const [copied, setCopied] = useState(false);+ const reading = useReadingThis(listen?.id ?? "");const time = formatTime(timestamp);return (- <div className={cx("-mb-2 flex h-6 items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100 focus-within:opacity-100", align === "end" ? "mt-0 flex-row-reverse" : "mt-2")}>+ <div className={cx("-mb-2 flex h-6 items-center gap-1 transition-opacity group-hover:opacity-100 focus-within:opacity-100", reading ? "opacity-100" : "opacity-0", align === "end" ? "mt-0 flex-row-reverse" : "mt-2")}>{copy && (<buttononClick={() => { void navigator.clipboard.writeText(copy).then(() => { setCopied(true); setTimeout(() => setCopied(false), 1200); }); }}@@ -112,6 +115,7 @@ function MessageMeta({ timestamp, copy, align = "start" }: { timestamp: string;{copied ? "Copied" : "Copy"}</button>)}+ {listen && <ReadAloudButton id={listen.id} text={listen.text} />}{time && <span className="px-1.5 text-xs text-ink4">{time}</span>}</div>);diff --git a/src/plugins/speak.ts b/src/plugins/speak.tsnew file mode 100644index 00000000..7d0e663c--- /dev/null+++ b/src/plugins/speak.ts@@ -0,0 +1,82 @@+/**+ * Text to speech, with the speech service the owner already set up (the+ * same one voice typing uses, see pickSpeechEngine): OpenAI+ * (gpt-4o-mini-tts, falling back to tts-1), Deepgram (Aura 2) or+ * ElevenLabs (Flash v2.5). Always MP3, so every client can play it.+ * Used to read Mercury's replies aloud in the apps.+ */+import type { SpeechEngine } from './transcribe.js';++/** Longest text spoken in one request; clients send a long reply in parts. */+export const SPEAK_MAX_CHARS = 4000;+/** Deepgram's own limit per request. */+const DEEPGRAM_MAX_CHARS = 2000;++const OPENAI_VOICE = 'coral';+const DEEPGRAM_MODEL = 'aura-2-thalia-en';+const ELEVENLABS_VOICE = 'JBFqnCBsd6RMkjVDRZzb';+const ELEVENLABS_MODEL = 'eleven_flash_v2_5';++type Fetch = typeof fetch;++export interface SpeakOptions {+ fetch?: Fetch;+ signal?: AbortSignal;+}++/** Speak `text` with `engine`; resolves with MP3 bytes. */+export async function synthesizeSpeech(text: string, engine: SpeechEngine, opts: SpeakOptions = {}): Promise<Uint8Array> {+ const input = text.trim();+ if (!input) throw new Error('nothing to say');+ const f = opts.fetch ?? fetch;+ const signal = opts.signal;++ if (engine.kind === 'deepgram') {+ const res = await f(`https://api.deepgram.com/v1/speak?model=${DEEPGRAM_MODEL}&encoding=mp3`, {+ method: 'POST',+ headers: { Authorization: `Token ${engine.apiKey}`, 'Content-Type': 'application/json' },+ body: JSON.stringify({ text: input.slice(0, DEEPGRAM_MAX_CHARS) }),+ signal,+ });+ return audioOf(res, 'Deepgram');+ }++ if (engine.kind === 'elevenlabs') {+ const res = await f(`https://api.elevenlabs.io/v1/text-to-speech/${ELEVENLABS_VOICE}?output_format=mp3_44100_128`, {+ method: 'POST',+ headers: { 'xi-api-key': engine.apiKey, 'Content-Type': 'application/json' },+ body: JSON.stringify({ text: input.slice(0, SPEAK_MAX_CHARS), model_id: ELEVENLABS_MODEL }),+ signal,+ });+ return audioOf(res, 'ElevenLabs');+ }++ const base = (engine.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');+ const say = (model: string) => f(`${base}/audio/speech`, {+ method: 'POST',+ headers: { Authorization: `Bearer ${engine.apiKey}`, 'Content-Type': 'application/json' },+ body: JSON.stringify({ model, voice: OPENAI_VOICE, input: input.slice(0, SPEAK_MAX_CHARS), response_format: 'mp3' }),+ signal,+ });+ let res = await say('gpt-4o-mini-tts');+ // An OpenAI-compatible endpoint without the newer model: try the classic one.+ if (res.status === 400 || res.status === 404) res = await say('tts-1');+ return audioOf(res, 'OpenAI');+}++async function audioOf(res: Response, vendor: string): Promise<Uint8Array> {+ if (!res.ok) {+ let detail = '';+ try {+ const body = (await res.json()) as { error?: { message?: string } | string; detail?: { message?: string } | string; err_msg?: string };+ const e = body.error ?? body.detail ?? body.err_msg;+ detail = typeof e === 'string' ? e : e?.message ?? '';+ } catch { /* not JSON */ }+ if (res.status === 401 || res.status === 403) throw new Error(`${vendor} rejected the key. Check it in Settings.`);+ if (res.status === 429) throw new Error(`${vendor} is rate-limiting right now. Try again in a moment.`);+ throw new Error(detail ? `${vendor}: ${detail.slice(0, 200)}` : `${vendor} answered ${res.status}`);+ }+ const bytes = new Uint8Array(await res.arrayBuffer());+ if (bytes.byteLength === 0) throw new Error(`${vendor} sent no audio`);+ return bytes;+}diff --git a/src/web/api-contract.ts b/src/web/api-contract.tsindex 05202ad0..9129a98b 100644--- a/src/web/api-contract.ts+++ b/src/web/api-contract.ts@@ -45,6 +45,7 @@ export const API_FEATURES: readonly string[] = Object.freeze(['chat.plan', // SSE 'plan' { steps, updatedAt } + GET /api/chat/plan?sessionId= (the plan checklist)'channels', // GET /api/channels, POST :type/check, PUT :type, POST :type/access, POST/GET signal/link'transcribe', // GET/POST /api/transcribe — dictation: raw audio in, text out+ 'speak', // GET/POST /api/speak — read aloud: text in, MP3 out'cloud.signin', // GET /api/cloud/status, POST/GET/DELETE /api/cloud/signin (browser pairing from any client) // POST /api/voice/check { vendor, key?, endpoint? }'brain.map', // GET /api/brain/map[?archive=1] — themes, sub-themes, notes, related links]);diff --git a/src/web/api/speak-route.test.ts b/src/web/api/speak-route.test.tsnew file mode 100644index 00000000..6a6005d4--- /dev/null+++ b/src/web/api/speak-route.test.ts@@ -0,0 +1,79 @@+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, describe, expect, it, vi } from 'vitest';++const home = await vi.hoisted(async () => {+ const fs = await import('node:fs');+ const os = await import('node:os');+ const path = await import('node:path');+ for (const k of ['OPENAI_API_KEY', 'DEEPGRAM_API_KEY', 'ELEVENLABS_API_KEY']) delete process.env[k];+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mercury-speak-'));+ fs.writeFileSync(path.join(dir, 'mercury.yaml'), 'identity:\n owner: Test\n');+ process.env.MERCURY_HOME = dir;+ return dir;+});++import speak, { setSpeakFetch } from './speak.js';++const mp3 = new Uint8Array([0xff, 0xfb, 0x90, 0x44, 1, 2, 3]);+const audioReply = () => ({ ok: true, status: 200, arrayBuffer: async () => mp3.buffer.slice(0) });+const post = (text: unknown) => speak.request('http://127.0.0.1/api/speak', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text }) });+const withOpenAI = () => writeFileSync(join(home, 'mercury.yaml'), [+ 'identity:', ' owner: Test', 'providers:', ' openai:', ' name: openai', ' apiKey: sk-test-openai-000000000000', ' baseUrl: https://api.openai.com/v1', ' model: gpt-4o-mini', ' enabled: true', '',+].join('\n'));++afterEach(() => { setSpeakFetch(undefined); delete process.env.DEEPGRAM_API_KEY; delete process.env.ELEVENLABS_API_KEY; });++describe('read aloud', () => {+ it('with no speech service, says how to turn one on', async () => {+ expect(await (await speak.request('http://127.0.0.1/api/speak')).json()).toEqual({ available: false, engine: null, maxChars: 4000 });+ const res = await post('Hello');+ expect(res.status).toBe(412);+ expect(await res.json()).toMatchObject({ needs: 'speech' });+ });++ it('speaks with the OpenAI provider and returns MP3', async () => {+ withOpenAI();+ const f = vi.fn().mockResolvedValue(audioReply());+ setSpeakFetch(f as never);+ const res = await post('The weather is sunny.');+ expect(res.status).toBe(200);+ expect(res.headers.get('content-type')).toBe('audio/mpeg');+ expect(new Uint8Array(await res.arrayBuffer())).toEqual(mp3);+ expect(f.mock.calls[0][0]).toBe('https://api.openai.com/v1/audio/speech');+ expect(JSON.parse(f.mock.calls[0][1].body)).toMatchObject({ model: 'gpt-4o-mini-tts', input: 'The weather is sunny.', response_format: 'mp3' });+ });++ it('falls back to tts-1 on an endpoint without the newer model', async () => {+ withOpenAI();+ const f = vi.fn().mockResolvedValueOnce({ ok: false, status: 404, json: async () => ({}) }).mockResolvedValueOnce(audioReply());+ setSpeakFetch(f as never);+ expect((await post('Hi')).status).toBe(200);+ expect(JSON.parse(f.mock.calls[1][1].body).model).toBe('tts-1');+ });++ it('uses a Deepgram or ElevenLabs voice key when there is no OpenAI', async () => {+ writeFileSync(join(home, 'mercury.yaml'), 'identity:\n owner: Test\n');+ process.env.ELEVENLABS_API_KEY = 'el-key';+ const f = vi.fn().mockResolvedValue(audioReply());+ setSpeakFetch(f as never);+ expect((await post('Hi there')).status).toBe(200);+ expect(f.mock.calls[0][0]).toContain('api.elevenlabs.io/v1/text-to-speech/');+ expect(f.mock.calls[0][1].headers['xi-api-key']).toBe('el-key');++ process.env.DEEPGRAM_API_KEY = 'dg-key';+ expect((await post('Hi there')).status).toBe(200);+ expect(f.mock.calls[1][0]).toContain('api.deepgram.com/v1/speak');+ expect(f.mock.calls[1][1].headers.Authorization).toBe('Token dg-key');+ });++ it('refuses empty or oversized text, and a rejected key reads plainly', async () => {+ withOpenAI();+ expect((await post(' ')).status).toBe(400);+ expect((await post('x'.repeat(4001))).status).toBe(413);+ setSpeakFetch(vi.fn().mockResolvedValue({ ok: false, status: 401, json: async () => ({ error: { message: 'bad key' } }) }) as never);+ const res = await post('Hello');+ expect(res.status).toBe(502);+ expect((await res.json() as any).error).toMatch(/rejected the key/);+ });+});diff --git a/src/web/api/speak.ts b/src/web/api/speak.tsnew file mode 100644index 00000000..26a37138--- /dev/null+++ b/src/web/api/speak.ts@@ -0,0 +1,53 @@+import { Hono } from 'hono';+import { loadConfig } from '../../utils/config.js';+import { secretValue } from '../../plugins/store.js';+import { pickSpeechEngine } from '../../plugins/transcribe.js';+import { SPEAK_MAX_CHARS, synthesizeSpeech } from '../../plugins/speak.js';++/**+ * Read aloud for every client: text in, speech out.+ *+ * GET /api/speak { available, engine, maxChars } — which speech service would be used+ * POST /api/speak { text } → audio/mpeg+ *+ * Uses the speech service voice typing uses (OpenAI provider, else a+ * Deepgram or ElevenLabs voice key). With none, 412 { needs: 'speech' } so+ * the app can say how to turn it on. Clients send a long reply in parts of+ * at most `maxChars`, so the first words play while the rest is made.+ */++type Fetch = typeof fetch;+let doFetch: Fetch | undefined;+/** Test seam. */+export function setSpeakFetch(f: Fetch | undefined): void { doFetch = f; }++function engine() {+ return pickSpeechEngine(loadConfig().providers.openai, secretValue);+}++const speak = new Hono();++speak.get('/api/speak', (c) => {+ const e = engine();+ return c.json({ available: !!e, engine: e?.label ?? null, maxChars: SPEAK_MAX_CHARS });+});++speak.post('/api/speak', async (c) => {+ const e = engine();+ if (!e) {+ return c.json({ error: 'Reading aloud needs a speech service: an OpenAI key (Settings → Models), or a Deepgram or ElevenLabs key (Settings → Voice).', needs: 'speech' }, 412);+ }+ const body = (await c.req.json().catch(() => ({}))) as { text?: unknown };+ const text = typeof body.text === 'string' ? body.text.trim() : '';+ if (!text) return c.json({ error: 'Nothing to read.' }, 400);+ if (text.length > SPEAK_MAX_CHARS) return c.json({ error: `Send at most ${SPEAK_MAX_CHARS} characters at a time.`, maxChars: SPEAK_MAX_CHARS }, 413);+ try {+ const audio = await synthesizeSpeech(text, e, { fetch: doFetch, signal: c.req.raw.signal });+ return new Response(audio, { headers: { 'Content-Type': 'audio/mpeg', 'Cache-Control': 'no-store', 'X-Speech-Engine': e.label } });+ } catch (err) {+ const message = err instanceof Error ? err.message : String(err);+ return c.json({ error: `Couldn't read that aloud: ${message}`, engine: e.label }, 502);+ }+});++export default speak;diff --git a/src/web/server.ts b/src/web/server.tsindex d64d157a..e47ad9dd 100644--- a/src/web/server.ts+++ b/src/web/server.ts@@ -14,6 +14,7 @@ import statusRoutes, { updateStatus } from './api/status.js';import providerRoutes from './api/providers.js';import cloudRoutes from './api/cloud.js';import transcribeRoutes from './api/transcribe.js';+import speakRoutes from './api/speak.js';import channelRoutes from './api/channels.js';import artifactRoutes from './api/artifacts.js';import configRoutes from './api/config.js';@@ -97,6 +98,7 @@ app.route('/', statusRoutes);app.route('/', providerRoutes);app.route('/', cloudRoutes);app.route('/', transcribeRoutes);+app.route('/', speakRoutes);app.route('/', channelRoutes);app.route('/', artifactRoutes);app.route('/', configRoutes);