Skip to content

Personal AI, On Personal Devices

Python

feat(server): speak assistant replies in the web UI (#963)

Rallxp4fadc42e8316
diff --git a/frontend/src/components/Chat/ChatArea.tsx b/frontend/src/components/Chat/ChatArea.tsx
index e0b2ffef..ed886b0e 100644
--- a/frontend/src/components/Chat/ChatArea.tsx
+++ b/frontend/src/components/Chat/ChatArea.tsx
@@ -4,6 +4,8 @@ import { MessageBubble } from './MessageBubble';
import { InputArea } from './InputArea';
import { StreamingDots } from './StreamingDots';
import { useAppStore } from '../../lib/store';
+import { shouldAutoplayFinishedReply, useTtsStore } from '../../lib/tts';
+import { stripThinkTags } from '../../lib/message-text';
import { Sparkles, PanelRightOpen, PanelRightClose, Database, MessageSquare, X } from 'lucide-react';
import { listConnectors } from '../../lib/connectors-api';
@@ -26,6 +28,63 @@ export function ChatArea() {
const wasStreaming = useRef(false);
const lastScrollTop = useRef(0);
const isCurrentChatStreaming = streamState.isStreaming && streamState.conversationId === activeId;
+
+ // Autoplay: speak a reply once it is finished, never while it streams -- a
+ // partial sentence would be synthesized and then cut off by the next chunk.
+ // autoSpokenId fences the message so re-renders cannot repeat it.
+ const voiceOutputEnabled = useAppStore((s) => s.settings.voiceOutputEnabled);
+ const voiceAutoplay = useAppStore((s) => s.settings.voiceAutoplay);
+ // Probe the backend as soon as voice output is switched on. Without this the
+ // first reply could never autoplay: `available` is only set by ensureHealth,
+ // which until now ran solely from the per-message read-aloud button -- and
+ // that button does not exist until a reply is already on screen.
+ useEffect(() => {
+ if (!voiceOutputEnabled) return;
+ useTtsStore.getState().ensureHealth();
+ }, [voiceOutputEnabled]);
+
+ // Speak on the falling edge of a stream -- the moment a reply finishes -- and
+ // never merely because a finished reply happens to be on screen. Opening the
+ // app or switching conversations would otherwise read stale history aloud,
+ // and browsers block that anyway: playback with no preceding user gesture is
+ // rejected outright, so the failure would be silent in both senses.
+ const streamingConversationRef = useRef<string | null>(null);
+ useEffect(() => {
+ const last = messages[messages.length - 1];
+ const tts = useTtsStore.getState();
+ const justFinished = shouldAutoplayFinishedReply(
+ streamingConversationRef.current,
+ activeId,
+ streamState.isStreaming,
+ last,
+ tts.autoSpokenId,
+ );
+ streamingConversationRef.current = isCurrentChatStreaming ? activeId : null;
+
+ if (!justFinished) return;
+ if (!voiceOutputEnabled || !voiceAutoplay) return;
+
+ if (!last || last.role !== 'assistant' || activeId === null) return;
+
+ // Model loading may still be in flight when a fast reply completes. Wait
+ // for the probe, then confirm that this is still the active finished reply.
+ const completedConversationId = activeId;
+ const completedMessageId = last.id;
+ void tts.ensureHealth().then(() => {
+ const app = useAppStore.getState();
+ const currentTts = useTtsStore.getState();
+ const currentLast = app.messages[app.messages.length - 1];
+ if (app.activeId !== completedConversationId || app.streamState.isStreaming) return;
+ if (!app.settings.voiceOutputEnabled || !app.settings.voiceAutoplay) return;
+ if (currentLast?.id !== completedMessageId || currentTts.available !== true) return;
+ if (currentTts.autoSpokenId === completedMessageId) return;
+
+ const text = stripThinkTags(currentLast.content);
+ if (!text) return;
+ currentTts.markAutoSpoken(completedMessageId);
+ void currentTts.speak(completedMessageId, text);
+ });
+ }, [activeId, messages, streamState.isStreaming, isCurrentChatStreaming, voiceOutputEnabled, voiceAutoplay]);
const currentStreamContent = isCurrentChatStreaming ? streamState.content : '';
// Check if any data sources are connected
diff --git a/frontend/src/components/Chat/MessageBubble.tsx b/frontend/src/components/Chat/MessageBubble.tsx
index 8a291d6c..320976cc 100644
--- a/frontend/src/components/Chat/MessageBubble.tsx
+++ b/frontend/src/components/Chat/MessageBubble.tsx
@@ -11,13 +11,9 @@ import { ToolCallCard } from './ToolCallCard';
import { ResearchTimeline } from './ResearchTimeline';
import { rehypeCitations } from '../../lib/rehype-citations';
import { XRayFooter } from './XRayFooter';
+import { SpeakMessageButton } from './SpeakMessageButton';
import type { ChatMessage } from '../../types';
-
-function stripThinkTags(text: string): string {
- let cleaned = text.replace(/<think>[\s\S]*?<\/think>\s*/gi, '');
- cleaned = cleaned.replace(/^[\s\S]*?<\/think>\s*/i, '');
- return cleaned.trim();
-}
+import { stripThinkTags } from '../../lib/message-text';
interface Props {
message: ChatMessage;
@@ -178,9 +174,12 @@ export function MessageBubble({ message, isLive = false }: Props) {
</div>
)}
- {/* Footer: copy + x-ray */}
+ {/* Footer: copy + read aloud + x-ray */}
<div className="flex items-center gap-2 mt-1.5">
<CopyMessageButton content={cleanContent} />
+ {!isUser && !isLive && (
+ <SpeakMessageButton messageId={message.id} content={cleanContent} />
+ )}
</div>
<XRayFooter
usage={message.usage}
diff --git a/frontend/src/components/Chat/SpeakMessageButton.test.tsx b/frontend/src/components/Chat/SpeakMessageButton.test.tsx
new file mode 100644
index 00000000..e1601be2
--- /dev/null
+++ b/frontend/src/components/Chat/SpeakMessageButton.test.tsx
@@ -0,0 +1,38 @@
+import { renderToStaticMarkup } from 'react-dom/server';
+import { describe, expect, it, vi } from 'vitest';
+
+vi.mock('../../lib/store', () => ({
+ useAppStore: (selector: (state: { settings: { voiceOutputEnabled: boolean } }) => unknown) =>
+ selector({ settings: { voiceOutputEnabled: true } }),
+}));
+vi.mock('../../hooks/useTts', () => ({ useTts: vi.fn() }));
+
+import { useTts } from '../../hooks/useTts';
+import { SpeakMessageButton } from './SpeakMessageButton';
+
+describe('read-aloud feedback', () => {
+ it('shows a playback error beside the message that failed', () => {
+ vi.mocked(useTts).mockReturnValue({
+ available: true,
+ state: 'idle',
+ speakingId: null,
+ error: 'Playback failed',
+ errorId: 'failed-message',
+ speak: vi.fn(async () => {}),
+ stop: vi.fn(),
+ isLoading: false,
+ isSpeaking: false,
+ });
+
+ const failed = renderToStaticMarkup(
+ <SpeakMessageButton messageId="failed-message" content="Hello" />,
+ );
+ const other = renderToStaticMarkup(
+ <SpeakMessageButton messageId="other-message" content="Hello" />,
+ );
+
+ expect(failed).toContain('role="alert"');
+ expect(failed).toContain('Playback failed');
+ expect(other).not.toContain('Playback failed');
+ });
+});
diff --git a/frontend/src/components/Chat/SpeakMessageButton.tsx b/frontend/src/components/Chat/SpeakMessageButton.tsx
new file mode 100644
index 00000000..cb36a57c
--- /dev/null
+++ b/frontend/src/components/Chat/SpeakMessageButton.tsx
@@ -0,0 +1,51 @@
+import { Volume2, Square } from 'lucide-react';
+import { useTts } from '../../hooks/useTts';
+import { useAppStore } from '../../lib/store';
+
+interface Props {
+ messageId: string;
+ content: string;
+}
+
+/** Reads one assistant message aloud through the server's TTS backend. */
+export function SpeakMessageButton({ messageId, content }: Props) {
+ const voiceOutputEnabled = useAppStore((s) => s.settings.voiceOutputEnabled);
+ const { speak, stop, speakingId, available, state, error, errorId } = useTts();
+
+ if (!voiceOutputEnabled || !available) return null;
+
+ // Playback is global, so this button reflects the shared utterance only when
+ // that utterance is this message.
+ const isMine = speakingId === messageId;
+ const busy = isMine && (state === 'loading' || state === 'speaking');
+ const messageError = errorId === messageId ? error : null;
+
+ return (
+ <>
+ <button
+ onClick={() => (busy ? stop() : speak(messageId, content))}
+ className="p-1 rounded opacity-0 group-hover:opacity-100 transition-opacity cursor-pointer"
+ style={{
+ color: busy ? 'var(--color-accent)' : 'var(--color-text-tertiary)',
+ opacity: busy ? 1 : undefined,
+ }}
+ title={
+ messageError ||
+ (isMine && state === 'loading'
+ ? 'Synthesizing...'
+ : isMine && state === 'speaking'
+ ? 'Stop'
+ : 'Read aloud')
+ }
+ aria-label={busy ? 'Stop reading message' : 'Read message aloud'}
+ >
+ {busy ? <Square size={14} /> : <Volume2 size={14} />}
+ </button>
+ {messageError && (
+ <span role="alert" className="text-xs" style={{ color: 'var(--color-error)' }}>
+ {messageError}
+ </span>
+ )}
+ </>
+ );
+}
diff --git a/frontend/src/hooks/useTts.ts b/frontend/src/hooks/useTts.ts
new file mode 100644
index 00000000..aec17703
--- /dev/null
+++ b/frontend/src/hooks/useTts.ts
@@ -0,0 +1,38 @@
+import { useEffect } from 'react';
+import { useTtsStore } from '../lib/tts';
+
+export type { TtsState } from '../lib/tts';
+
+/**
+ * Read-aloud controls backed by the shared voice-output store.
+ *
+ * Every consumer talks to the same utterance, so starting one reply stops the
+ * previous one instead of layering on top of it. The health probe runs once per
+ * document, no matter how many components mount this hook.
+ */
+export function useTts() {
+ const state = useTtsStore((s) => s.state);
+ const speakingId = useTtsStore((s) => s.speakingId);
+ const error = useTtsStore((s) => s.error);
+ const errorId = useTtsStore((s) => s.errorId);
+ const available = useTtsStore((s) => s.available);
+ const speak = useTtsStore((s) => s.speak);
+ const stop = useTtsStore((s) => s.stop);
+ const ensureHealth = useTtsStore((s) => s.ensureHealth);
+
+ useEffect(() => {
+ ensureHealth();
+ }, [ensureHealth]);
+
+ return {
+ state,
+ speakingId,
+ error,
+ errorId,
+ available: available === true,
+ speak,
+ stop,
+ isLoading: state === 'loading',
+ isSpeaking: state === 'speaking',
+ };
+}
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index 0cf9548e..03bc8ca7 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -388,6 +388,43 @@ export async function transcribeAudio(audioBlob: Blob, filename = 'recording.web
return res.json();
}
+export interface TtsHealth {
+ available: boolean;
+ backend?: string;
+ voice_id?: string;
+ speed?: number;
+ reason?: string;
+}
+
+export async function synthesizeSpeech(
+ text: string,
+ opts: { voiceId?: string; speed?: number; signal?: AbortSignal } = {},
+): Promise<Blob> {
+ const res = await apiFetch(`/v1/speech/synthesize`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ text, voice_id: opts.voiceId, speed: opts.speed }),
+ signal: opts.signal,
+ });
+ if (!res.ok) {
+ let detail = '';
+ try {
+ const body = await res.json();
+ detail = typeof body.detail === 'string' ? body.detail : '';
+ } catch {
+ // Keep the status-only message below when the body is not JSON.
+ }
+ throw new Error(detail || `Speech synthesis failed: ${res.status}`);
+ }
+ return res.blob();
+}
+
+export async function fetchTtsHealth(): Promise<TtsHealth> {
+ const res = await apiFetch(`/v1/speech/tts/health`);
+ if (!res.ok) return { available: false };
+ return res.json();
+}
+
export async function fetchSpeechHealth(): Promise<SpeechHealth> {
if (isTauri()) {
try {
diff --git a/frontend/src/lib/message-text.ts b/frontend/src/lib/message-text.ts
new file mode 100644
index 00000000..e994bc02
--- /dev/null
+++ b/frontend/src/lib/message-text.ts
@@ -0,0 +1,8 @@
+/** Text helpers shared between the message renderer and voice output. */
+
+/** Remove <think> reasoning blocks so they are neither shown nor spoken. */
+export function stripThinkTags(text: string): string {
+ let cleaned = text.replace(/<think>[\s\S]*?<\/think>\s*/gi, '');
+ cleaned = cleaned.replace(/^[\s\S]*?<\/think>\s*/i, '');
+ return cleaned.trim();
+}
diff --git a/frontend/src/lib/store.ts b/frontend/src/lib/store.ts
index 4f45b042..dcb1ffc6 100644
--- a/frontend/src/lib/store.ts
+++ b/frontend/src/lib/store.ts
@@ -106,6 +106,8 @@ interface Settings {
temperature: number;
maxTokens: number;
speechEnabled: boolean;
+ voiceOutputEnabled: boolean;
+ voiceAutoplay: boolean;
}
function loadSettings(): Settings {
@@ -119,6 +121,8 @@ function loadSettings(): Settings {
temperature: 0.7,
maxTokens: 4096,
speechEnabled: false,
+ voiceOutputEnabled: false,
+ voiceAutoplay: false,
};
try {
const raw = localStorage.getItem(SETTINGS_KEY);
diff --git a/frontend/src/lib/tts.test.ts b/frontend/src/lib/tts.test.ts
new file mode 100644
index 00000000..f52d84d1
--- /dev/null
+++ b/frontend/src/lib/tts.test.ts
@@ -0,0 +1,254 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('./api', () => ({
+ synthesizeSpeech: vi.fn(),
+ fetchTtsHealth: vi.fn(),
+}));
+
+import { synthesizeSpeech, fetchTtsHealth } from './api';
+import { useTtsStore, __resetTtsForTests, shouldAutoplayFinishedReply } from './tts';
+
+const synth = vi.mocked(synthesizeSpeech);
+const health = vi.mocked(fetchTtsHealth);
+
+/** Minimal HTMLAudioElement stand-in; jsdom cannot decode or play real audio. */
+class FakeAudio {
+ static instances: FakeAudio[] = [];
+ onended: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+ src: string;
+ paused = false;
+ playCalls = 0;
+
+ constructor(src: string) {
+ this.src = src;
+ FakeAudio.instances.push(this);
+ }
+
+ play(): Promise<void> {
+ this.playCalls += 1;
+ return Promise.resolve();
+ }
+
+ pause(): void {
+ this.paused = true;
+ }
+}
+
+const created: string[] = [];
+const revoked: string[] = [];
+let urlCounter = 0;
+
+beforeEach(() => {
+ vi.stubGlobal('Audio', FakeAudio as unknown as typeof Audio);
+ vi.stubGlobal('URL', {
+ createObjectURL: () => {
+ const url = `blob:fake/${++urlCounter}`;
+ created.push(url);
+ return url;
+ },
+ revokeObjectURL: (url: string) => {
+ revoked.push(url);
+ },
+ });
+
+ synth.mockReset();
+ health.mockReset();
+
+ // Reset the store first: it tears down whatever the previous test left
+ // playing, and that teardown revokes a URL. Clearing the ledgers afterwards
+ // keeps that bookkeeping out of this test's assertions.
+ __resetTtsForTests();
+ FakeAudio.instances = [];
+ created.length = 0;
+ revoked.length = 0;
+ urlCounter = 0;
+});
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+function deferred<T>() {
+ let resolve!: (value: T) => void;
+ let reject!: (reason?: unknown) => void;
+ const promise = new Promise<T>((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+ return { promise, resolve, reject };
+}
+
+describe('shared voice output', () => {
+ it('plays one utterance and reports which message owns it', async () => {
+ synth.mockResolvedValue(new Blob(['wav']));
+
+ await useTtsStore.getState().speak('m1', 'Hallo');
+
+ expect(FakeAudio.instances).toHaveLength(1);
+ expect(FakeAudio.instances[0].playCalls).toBe(1);
+ expect(useTtsStore.getState().state).toBe('speaking');
+ expect(useTtsStore.getState().speakingId).toBe('m1');
+ });
+
+ it('starting a second message stops the first instead of overlapping it', async () => {
+ synth.mockResolvedValue(new Blob(['wav']));
+
+ await useTtsStore.getState().speak('m1', 'Erste');
+ const first = FakeAudio.instances[0];
+
+ await useTtsStore.getState().speak('m2', 'Zweite');
+
+ expect(first.paused).toBe(true);
+ expect(useTtsStore.getState().speakingId).toBe('m2');
+ expect(revoked).toContain(created[0]);
+ });
+
+ it('aborts a synthesis that is superseded before it resolves', async () => {
+ const pending = deferred<Blob>();
+ synth.mockReturnValueOnce(pending.promise);
+ synth.mockResolvedValueOnce(new Blob(['wav']));
+
+ const firstCall = useTtsStore.getState().speak('m1', 'Erste');
+ const signal = synth.mock.calls[0][1]?.signal as AbortSignal;
+ expect(signal.aborted).toBe(false);
+
+ await useTtsStore.getState().speak('m2', 'Zweite');
+ expect(signal.aborted).toBe(true);
+
+ // The stale response must not start playback or leak its blob URL.
+ pending.resolve(new Blob(['stale']));
+ await firstCall;
+
+ expect(FakeAudio.instances).toHaveLength(1);
+ expect(useTtsStore.getState().speakingId).toBe('m2');
+ });
+
+ it('stop() halts playback and revokes the blob URL', async () => {
+ synth.mockResolvedValue(new Blob(['wav']));
+
+ await useTtsStore.getState().speak('m1', 'Hallo');
+ useTtsStore.getState().stop();
+
+ expect(FakeAudio.instances[0].paused).toBe(true);
+ expect(revoked).toEqual(created);
+ expect(useTtsStore.getState().state).toBe('idle');
+ expect(useTtsStore.getState().speakingId).toBeNull();
+ });
+
+ it('finishing playback leaves no error behind', async () => {
+ synth.mockResolvedValue(new Blob(['wav']));
+
+ await useTtsStore.getState().speak('m1', 'Hallo');
+ const el = FakeAudio.instances[0];
+
+ el.onended?.();
+ // Clearing src fires a media error; the handler must already be detached.
+ el.onerror?.();
+
+ expect(useTtsStore.getState().error).toBeNull();
+ expect(useTtsStore.getState().state).toBe('idle');
+ expect(revoked).toEqual(created);
+ });
+
+ it('surfaces a synthesis failure', async () => {
+ synth.mockRejectedValue(new Error('No text-to-speech backend available'));
+
+ await useTtsStore.getState().speak('m1', 'Hallo');
+
+ expect(useTtsStore.getState().error).toBe('No text-to-speech backend available');
+ expect(useTtsStore.getState().errorId).toBe('m1');
+ expect(useTtsStore.getState().state).toBe('idle');
+ });
+
+ it('ignores empty text', async () => {
+ await useTtsStore.getState().speak('m1', ' ');
+ expect(synth).not.toHaveBeenCalled();
+ });
+
+ it('probes the backend once no matter how many callers ask', () => {
+ health.mockResolvedValue({ available: true });
+
+ useTtsStore.getState().ensureHealth();
+ useTtsStore.getState().ensureHealth();
+ useTtsStore.getState().ensureHealth();
+
+ expect(health).toHaveBeenCalledTimes(1);
+ });
+
+ it('shares a pending health probe so autoplay can wait for model loading', async () => {
+ const pending = deferred<{ available: boolean }>();
+ health.mockReturnValue(pending.promise);
+
+ const first = useTtsStore.getState().ensureHealth();
+ const second = useTtsStore.getState().ensureHealth();
+ expect(second).toBe(first);
+ expect(health).toHaveBeenCalledTimes(1);
+
+ pending.resolve({ available: true });
+ await first;
+ expect(useTtsStore.getState().available).toBe(true);
+ });
+
+ it('retries health after an unavailable response', async () => {
+ health.mockResolvedValueOnce({ available: false });
+ health.mockResolvedValueOnce({ available: true });
+
+ await useTtsStore.getState().ensureHealth();
+ expect(useTtsStore.getState().available).toBe(false);
+ await useTtsStore.getState().ensureHealth();
+
+ expect(health).toHaveBeenCalledTimes(2);
+ expect(useTtsStore.getState().available).toBe(true);
+ });
+
+ it('retries health after a network error', async () => {
+ health.mockRejectedValueOnce(new Error('offline'));
+ health.mockResolvedValueOnce({ available: true });
+
+ await useTtsStore.getState().ensureHealth();
+ expect(useTtsStore.getState().available).toBe(false);
+ await useTtsStore.getState().ensureHealth();
+
+ expect(health).toHaveBeenCalledTimes(2);
+ expect(useTtsStore.getState().available).toBe(true);
+ });
+});
+
+describe('autoplay gating', () => {
+ it('stays silent when a finished conversation is merely restored', () => {
+ expect(
+ shouldAutoplayFinishedReply(null, 'chat-1', false, { role: 'assistant', id: 'old' }, null),
+ ).toBe(false);
+ });
+
+ it('speaks when a stream finishes', () => {
+ expect(
+ shouldAutoplayFinishedReply('chat-1', 'chat-1', false, { role: 'assistant', id: 'fresh' }, null),
+ ).toBe(true);
+ });
+
+ it('stays silent while the reply is still streaming', () => {
+ expect(
+ shouldAutoplayFinishedReply('chat-1', 'chat-1', true, { role: 'assistant', id: 'fresh' }, null),
+ ).toBe(false);
+ });
+
+ it('never repeats a message it already spoke', () => {
+ expect(
+ shouldAutoplayFinishedReply('chat-1', 'chat-1', false, { role: 'assistant', id: 'fresh' }, 'fresh'),
+ ).toBe(false);
+ });
+
+ it('ignores a trailing user message', () => {
+ expect(
+ shouldAutoplayFinishedReply('chat-1', 'chat-1', false, { role: 'user', id: 'u1' }, null),
+ ).toBe(false);
+ });
+
+ it('does not speak an old reply when the user switches conversations mid-stream', () => {
+ expect(
+ shouldAutoplayFinishedReply('chat-1', 'chat-2', true, { role: 'assistant', id: 'old' }, null),
+ ).toBe(false);
+ });
+});
diff --git a/frontend/src/lib/tts.ts b/frontend/src/lib/tts.ts
new file mode 100644
index 00000000..b331e86a
--- /dev/null
+++ b/frontend/src/lib/tts.ts
@@ -0,0 +1,169 @@
+import { create } from 'zustand';
+import { synthesizeSpeech, fetchTtsHealth } from './api';
+
+export type TtsState = 'idle' | 'loading' | 'speaking';
+
+/**
+ * Voice output is a single shared resource: one utterance at a time, for the
+ * whole app. The state lives in one store rather than per component, because a
+ * per-component hook would give every message its own audio element -- two
+ * replies would then talk over each other, and autoplay would make that the
+ * normal case rather than the exception.
+ */
+interface TtsStore {
+ state: TtsState;
+ /** id of the message currently loading or speaking, if any. */
+ speakingId: string | null;
+ error: string | null;
+ /** Message whose read-aloud attempt failed, if any. */
+ errorId: string | null;
+ /** null until the health probe has answered. */
+ available: boolean | null;
+ /** Last message spoken by autoplay, so a re-render never repeats it. */
+ autoSpokenId: string | null;
+ speak: (id: string, text: string) => Promise<void>;
+ stop: () => void;
+ ensureHealth: () => Promise<void>;
+ markAutoSpoken: (id: string) => void;
+}
+
+// Playback handles are not render state -- keeping them out of the store avoids
+// re-rendering every subscriber when an audio element is swapped.
+let audio: HTMLAudioElement | null = null;
+let objectUrl: string | null = null;
+let controller: AbortController | null = null;
+let token = 0;
+let healthProbe: Promise<void> | null = null;
+
+/** Only a stream ending in the active conversation may trigger autoplay. */
+export function shouldAutoplayFinishedReply(
+ previousStreamingConversationId: string | null,
+ activeId: string | null,
+ streamIsActive: boolean,
+ lastMessage: { id: string; role: string } | undefined,
+ autoSpokenId: string | null,
+): boolean {
+ return previousStreamingConversationId !== null
+ && previousStreamingConversationId === activeId
+ && !streamIsActive
+ && lastMessage !== undefined
+ && lastMessage.role === 'assistant'
+ && lastMessage.id !== autoSpokenId;
+}
+
+function teardown(): void {
+ if (audio) {
+ // Detach first: clearing src re-runs the media load algorithm, which fails
+ // on an empty source and dispatches an `error` event. With the handler
+ // still attached that surfaces as a bogus "Playback failed" after every
+ // successful utterance.
+ audio.onended = null;
+ audio.onerror = null;
+ audio.pause();
+ audio.src = '';
+ audio = null;
+ }
+ if (objectUrl) {
+ URL.revokeObjectURL(objectUrl);
+ objectUrl = null;
+ }
+ if (controller) {
+ controller.abort();
+ controller = null;
+ }
+}
+
+export const useTtsStore = create<TtsStore>((set, get) => ({
+ state: 'idle',
+ speakingId: null,
+ error: null,
+ errorId: null,
+ available: null,
+ autoSpokenId: null,
+
+ ensureHealth: () => {
+ if (healthProbe) return healthProbe;
+ healthProbe = fetchTtsHealth()
+ .then((health) => {
+ set({ available: health.available });
+ if (!health.available) healthProbe = null;
+ })
+ .catch(() => {
+ set({ available: false });
+ healthProbe = null;
+ });
+ return healthProbe;
+ },
+
+ markAutoSpoken: (id: string) => set({ autoSpokenId: id }),
+
+ speak: async (id: string, text: string) => {
+ const trimmed = text.trim();
+ if (!trimmed) return;
+
+ // Bump before teardown so a synthesis still in flight is both aborted and
+ // fenced off by the token, even if the abort loses the race.
+ token += 1;
+ const mine = token;
+ teardown();
+ set({ state: 'loading', speakingId: id, error: null, errorId: null });
+
+ const ac = new AbortController();
+ controller = ac;
+
+ try {
+ const blob = await synthesizeSpeech(trimmed, { signal: ac.signal });
+ if (mine !== token) return;
+
+ const url = URL.createObjectURL(blob);
+ objectUrl = url;
+ const el = new Audio(url);
+ audio = el;
+
+ el.onended = () => {
+ if (mine !== token) return;
+ teardown();
+ set({ state: 'idle', speakingId: null });
+ };
+ el.onerror = () => {
+ if (mine !== token) return;
+ teardown();
+ set({ state: 'idle', speakingId: null, error: 'Playback failed', errorId: id });
+ };
+
+ await el.play();
+ if (mine === token) set({ state: 'speaking', speakingId: id });
+ } catch (err) {
+ if (mine !== token) return;
+ if (err instanceof DOMException && err.name === 'AbortError') return;
+ teardown();
+ set({
+ state: 'idle',
+ speakingId: null,
+ error: err instanceof Error ? err.message : 'Speech synthesis failed',
+ errorId: id,
+ });
+ }
+ },
+
+ stop: () => {
+ token += 1;
+ teardown();
+ set({ state: 'idle', speakingId: null, error: null, errorId: null });
+ },
+}));
+
+/** Test seam: reset module-level playback handles between cases. */
+export function __resetTtsForTests(): void {
+ teardown();
+ token = 0;
+ healthProbe = null;
+ useTtsStore.setState({
+ state: 'idle',
+ speakingId: null,
+ error: null,
+ errorId: null,
+ available: null,
+ autoSpokenId: null,
+ });
+}
diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx
index fc207f7e..2437b299 100644
--- a/frontend/src/pages/SettingsPage.tsx
+++ b/frontend/src/pages/SettingsPage.tsx
@@ -22,6 +22,7 @@ import { useAppStore, type ThemeMode } from '../lib/store';
import {
checkHealth,
fetchSpeechHealth,
+ fetchTtsHealth,
getMemoryStats,
getInferenceSource,
setInferenceSource,
@@ -240,6 +241,7 @@ export function SettingsPage() {
const serverInfo = useAppStore((s) => s.serverInfo);
const [healthy, setHealthy] = useState<boolean | null>(null);
const [speechBackendAvailable, setSpeechBackendAvailable] = useState<boolean | null>(null);
+ const [ttsBackend, setTtsBackend] = useState<{ available: boolean; backend?: string; voice_id?: string } | null>(null);
const [saved, setSaved] = useState(false);
const [autoUpdateEnabled, setAutoUpdateEnabled] = useState(() => !isAutoUpdateDisabled());
@@ -314,6 +316,9 @@ export function SettingsPage() {
fetchSpeechHealth()
.then((h) => setSpeechBackendAvailable(h.available))
.catch(() => setSpeechBackendAvailable(false));
+ fetchTtsHealth()
+ .then((h) => setTtsBackend(h))
+ .catch(() => setTtsBackend({ available: false }));
getMemoryStats()
.then(setMemoryStats)
.catch(() => setMemoryStats(null));
@@ -722,6 +727,58 @@ export function SettingsPage() {
/>
</button>
</SettingRow>
+ <SettingRow label="Text-to-Speech" description="Show a read-aloud button on assistant replies">
+ <button
+ onClick={() => { updateSettings({ voiceOutputEnabled: !settings.voiceOutputEnabled }); showSaved(); }}
+ className="relative w-11 h-6 rounded-full transition-colors cursor-pointer"
+ style={{
+ background: settings.voiceOutputEnabled ? 'var(--color-accent)' : 'var(--color-bg-tertiary)',
+ }}
+ >
+ <span
+ className="absolute top-0.5 left-0.5 w-5 h-5 rounded-full transition-transform bg-white"
+ style={{
+ transform: settings.voiceOutputEnabled ? 'translateX(20px)' : 'translateX(0)',
+ boxShadow: '0 1px 3px rgba(0,0,0,0.2)',
+ }}
+ />
+ </button>
+ </SettingRow>
+ <SettingRow label="Speak replies automatically" description="Read every assistant reply aloud as soon as it finishes">
+ <button
+ onClick={() => { updateSettings({ voiceAutoplay: !settings.voiceAutoplay }); showSaved(); }}
+ disabled={!settings.voiceOutputEnabled}
+ className="relative w-11 h-6 rounded-full transition-colors cursor-pointer"
+ style={{
+ background: settings.voiceAutoplay && settings.voiceOutputEnabled ? 'var(--color-accent)' : 'var(--color-bg-tertiary)',
+ opacity: settings.voiceOutputEnabled ? 1 : 0.4,
+ cursor: settings.voiceOutputEnabled ? 'pointer' : 'default',
+ }}
+ >
+ <span
+ className="absolute top-0.5 left-0.5 w-5 h-5 rounded-full transition-transform bg-white"
+ style={{
+ transform: settings.voiceAutoplay && settings.voiceOutputEnabled ? 'translateX(20px)' : 'translateX(0)',
+ boxShadow: '0 1px 3px rgba(0,0,0,0.2)',
+ }}
+ />
+ </button>
+ </SettingRow>
+ <SettingRow label="Voice" description="Backend and voice used for spoken replies">
+ <div className="flex items-center gap-2">
+ <span
+ className="w-2 h-2 rounded-full"
+ style={{
+ background: ttsBackend?.available ? 'var(--color-success)' : 'var(--color-text-tertiary)',
+ }}
+ />
+ <span className="text-xs" style={{ color: 'var(--color-text-secondary)' }}>
+ {ttsBackend === null ? 'Checking...'
+ : ttsBackend.available ? `${ttsBackend.backend}${ttsBackend.voice_id ? ` / ${ttsBackend.voice_id}` : ''}`
+ : 'Not configured'}
+ </span>
+ </div>
+ </SettingRow>
<SettingRow label="Backend status" description="Requires Whisper, Deepgram, or another speech backend">
<div className="flex items-center gap-2">
<span
diff --git a/src/openjarvis/cli/_voice_chat.py b/src/openjarvis/cli/_voice_chat.py
index 3ce65237..b4623080 100644
--- a/src/openjarvis/cli/_voice_chat.py
+++ b/src/openjarvis/cli/_voice_chat.py
@@ -7,17 +7,26 @@ from typing import Any, Optional
from rich.markup import escape
VOICE_EXIT = object()
-_TTS_BACKEND_ORDER = ("kokoro", "openai_tts", "cartesia")
-# Voice IDs are backend-specific and NOT portable. ``speech.voice_id`` applies
-# only to ``speech.tts_backend``; if synthesis falls back to another backend we
-# use that backend's own default rather than passing an unrecognized ID through.
-_BACKEND_DEFAULT_VOICE = {
- "kokoro": "bm_george", # British male
- "openai_tts": "onyx", # deepest OpenAI preset
- "cartesia": "", # no safe static default; let Cartesia choose
+
+# Backend selection is shared with the API server and lives in
+# openjarvis.speech._tts_discovery. Importing that at module level would pull
+# the whole speech stack -- numpy included -- into every `jarvis` invocation,
+# which tests/cli/test_cli.py guards against, so these aliases resolve lazily.
+_LAZY_TTS_NAMES = {
+ "_TTS_BACKEND_ORDER": "TTS_BACKEND_ORDER",
+ "_BACKEND_DEFAULT_VOICE": "BACKEND_DEFAULT_VOICE",
}
+def __getattr__(name: str) -> Any:
+ target = _LAZY_TTS_NAMES.get(name)
+ if target is None:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+ from openjarvis.speech import _tts_discovery
+
+ return getattr(_tts_discovery, target)
+
+
def _terminal_safe_text(value: object) -> str:
"""Remove terminal controls and escape Rich markup from dynamic text."""
printable = "".join(
@@ -54,40 +63,20 @@ class VoiceSession:
if self._tts_backend is not None:
return self._tts_backend
- # Import triggers built-in backend registration only when voice output
- # is actually requested.
- import openjarvis.speech # noqa: F401
- from openjarvis.core.registry import TTSRegistry
+ from openjarvis.speech._tts_discovery import get_tts_backend
preferred, _, _ = self.get_voice_preferences()
- backend_order = dict.fromkeys((preferred, *_TTS_BACKEND_ORDER))
- for key in backend_order:
- if key in self._tts_attempted:
- continue
- self._tts_attempted.add(key)
- if not TTSRegistry.contains(key):
- continue
- try:
- candidate = TTSRegistry.get(key)()
- if candidate.health():
- self._tts_backend = candidate
- return candidate
- except Exception:
- continue
- return None
+ self._tts_backend = get_tts_backend(preferred, attempted=self._tts_attempted)
+ return self._tts_backend
def get_voice_preferences(self) -> tuple[str, str, float]:
"""Resolve configured (tts_backend, voice_id, speed), cached per session."""
if self._voice_prefs is None:
from openjarvis.core.config import load_config
+ from openjarvis.speech._tts_discovery import voice_preferences
config = self._config if self._config is not None else load_config()
- speech = getattr(config, "speech", None)
- self._voice_prefs = (
- getattr(speech, "tts_backend", "kokoro") or "kokoro",
- getattr(speech, "voice_id", "") or "",
- float(getattr(speech, "voice_speed", 1.0)),
- )
+ self._voice_prefs = voice_preferences(config)
return self._voice_prefs
def voice_for_backend(self, backend: Any, console: Any = None) -> tuple[str, float]:
@@ -103,7 +92,9 @@ class VoiceSession:
if active == want_backend:
return voice_id, speed
- substitute = _BACKEND_DEFAULT_VOICE.get(active, "")
+ from openjarvis.speech._tts_discovery import default_voice_for
+
+ substitute = default_voice_for(active)
if console is not None and active not in self._voice_warned:
self._voice_warned.add(active)
console.print(
diff --git a/src/openjarvis/server/api_routes.py b/src/openjarvis/server/api_routes.py
index 87b8cea1..27dc4419 100644
--- a/src/openjarvis/server/api_routes.py
+++ b/src/openjarvis/server/api_routes.py
@@ -9,7 +9,14 @@ import logging
from pathlib import Path
from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, HTTPException, Request, WebSocket, WebSocketDisconnect
+from fastapi import (
+ APIRouter,
+ HTTPException,
+ Request,
+ Response,
+ WebSocket,
+ WebSocketDisconnect,
+)
from pydantic import BaseModel
logger = logging.getLogger(__name__)
@@ -27,6 +34,12 @@ class AgentMessageRequest(BaseModel):
message: str
+class SpeechSynthesizeRequest(BaseModel):
+ text: str
+ voice_id: Optional[str] = None
+ speed: Optional[float] = None
+
+
class MemoryStoreRequest(BaseModel):
content: str
metadata: Optional[Dict[str, Any]] = None
@@ -1044,6 +1057,139 @@ async def speech_health(request: Request):
}
+# Synthesis is CPU-bound and runs in a worker thread; cap the input so one
+# request cannot occupy that thread for minutes.
+_MAX_TTS_CHARS = 5000
+
+
+async def _resolve_tts_backend(request: Request):
+ """Return a healthy TTS backend, resolved once and cached on app state.
+
+ Backends load models on construction, so this stays lazy: a server whose
+ users never ask for audio never pays for it.
+ """
+ app = request.app
+ if getattr(app.state, "tts_resolved", False):
+ return getattr(app.state, "tts_backend", None)
+
+ # The first health probe can load Kokoro's model. A shared task keeps that
+ # work off the event loop and coalesces concurrent probes, including when
+ # the backend is unavailable. A later request may retry that failure.
+ task = getattr(app.state, "tts_resolution_task", None)
+ if task is None:
+
+ def discover():
+ from openjarvis.speech._tts_discovery import (
+ get_tts_backend,
+ voice_preferences,
+ )
+
+ config = getattr(app.state, "config", None)
+ if config is None:
+ from openjarvis.core.config import load_config
+
+ config = load_config()
+
+ preferred, _, _ = voice_preferences(config)
+ return get_tts_backend(preferred)
+
+ task = asyncio.create_task(asyncio.to_thread(discover))
+ app.state.tts_resolution_task = task
+
+ try:
+ backend = await asyncio.shield(task)
+ finally:
+ if task.done() and getattr(app.state, "tts_resolution_task", None) is task:
+ app.state.tts_resolution_task = None
+
+ if backend is not None:
+ app.state.tts_backend = backend
+ app.state.tts_resolved = True
+ return backend
+
+
+def _tts_voice_and_speed(request: Request, backend) -> tuple[str, float]:
+ """Resolve the voice ID and speed to use with *backend*.
+
+ Voice IDs are not portable between backends, so a configured ``voice_id``
+ applies only when the resolved backend is the configured one; otherwise the
+ fallback backend's own default is used.
+ """
+ from openjarvis.speech._tts_discovery import default_voice_for, voice_preferences
+
+ config = getattr(request.app.state, "config", None)
+ if config is None:
+ return "", 1.0
+
+ preferred, voice_id, speed = voice_preferences(config)
+ active = getattr(backend, "backend_id", "")
+ if active and active != preferred:
+ voice_id = default_voice_for(active)
+ return voice_id, speed
+
+
+@speech_router.post("/synthesize")
+async def synthesize_speech(request: Request, body: SpeechSynthesizeRequest):
+ """Synthesize text to speech and return the audio as WAV."""
+ text = body.text.strip()
+ if not text:
+ raise HTTPException(status_code=400, detail="Missing 'text'")
+ if len(text) > _MAX_TTS_CHARS:
+ raise HTTPException(
+ status_code=413,
+ detail=f"Text exceeds {_MAX_TTS_CHARS} characters",
+ )
+
+ backend = await _resolve_tts_backend(request)
+ if backend is None:
+ raise HTTPException(
+ status_code=501, detail="No text-to-speech backend available"
+ )
+
+ voice_id, speed = _tts_voice_and_speed(request, backend)
+ if body.voice_id:
+ voice_id = body.voice_id
+ if body.speed is not None:
+ speed = body.speed
+
+ try:
+ result = await asyncio.to_thread(
+ backend.synthesize,
+ text,
+ voice_id=voice_id,
+ speed=speed,
+ output_format="wav",
+ )
+ except Exception as exc:
+ logger.exception("Speech synthesis failed")
+ raise HTTPException(status_code=500, detail="Speech synthesis failed") from exc
+
+ return Response(
+ content=result.audio,
+ media_type="audio/wav",
+ headers={
+ "X-Voice-Id": result.voice_id or voice_id,
+ "X-Sample-Rate": str(result.sample_rate),
+ },
+ )
+
+
+@speech_router.get("/tts/health")
+async def tts_health(request: Request):
+ """Report whether voice output is available, and with which voice."""
+ backend = await _resolve_tts_backend(request)
+ if backend is None:
+ return {"available": False, "reason": "No text-to-speech backend available"}
+
+ voice_id, speed = _tts_voice_and_speed(request, backend)
+ return {
+ "available": True,
+ "backend": getattr(backend, "backend_id", ""),
+ "voice_id": voice_id,
+ "speed": speed,
+ }
+
+
# ---- Feedback routes ----
feedback_router = APIRouter(prefix="/v1/feedback", tags=["feedback"])
diff --git a/src/openjarvis/server/middleware.py b/src/openjarvis/server/middleware.py
index 4031f287..df74cc09 100644
--- a/src/openjarvis/server/middleware.py
+++ b/src/openjarvis/server/middleware.py
@@ -45,11 +45,16 @@ def create_security_middleware() -> Any:
"max-age=31536000; includeSubDomains"
)
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+ # microphone=(self): the chat UI records voice input from this same
+ # origin. An empty allowlist blocked its own feature.
response.headers["Permissions-Policy"] = (
- "camera=(), microphone=(), geolocation=()"
+ "camera=(), microphone=(self), geolocation=()"
)
+ # media-src blob: lets the UI play synthesized speech, which arrives
+ # as a blob URL; default-src alone refused it.
response.headers["Content-Security-Policy"] = (
- "default-src 'self' 'unsafe-inline' 'unsafe-eval'"
+ "default-src 'self' 'unsafe-inline' 'unsafe-eval'; "
+ "media-src 'self' blob:"
)
return response
@@ -63,6 +68,8 @@ SECURITY_HEADERS = {
"X-XSS-Protection": "1; mode=block",
"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
"Referrer-Policy": "strict-origin-when-cross-origin",
- "Permissions-Policy": "camera=(), microphone=(), geolocation=()",
- "Content-Security-Policy": "default-src 'self' 'unsafe-inline' 'unsafe-eval'",
+ "Permissions-Policy": "camera=(), microphone=(self), geolocation=()",
+ "Content-Security-Policy": (
+ "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' blob:"
+ ),
}
diff --git a/src/openjarvis/speech/_tts_discovery.py b/src/openjarvis/speech/_tts_discovery.py
new file mode 100644
index 00000000..b77b9615
--- /dev/null
+++ b/src/openjarvis/speech/_tts_discovery.py
@@ -0,0 +1,85 @@
+"""Auto-discover available text-to-speech backends.
+
+Mirrors ``speech/_discovery.py``, which does the same for speech-to-text. The
+selection rules used to live in ``cli/_voice_chat.py``; they moved here so the
+API server can reuse them instead of importing from the CLI package.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+ from openjarvis.core.config import JarvisConfig
+
+logger = logging.getLogger(__name__)
+
+# Priority order: local first, then cloud.
+TTS_BACKEND_ORDER = ("kokoro", "openai_tts", "cartesia")
+
+# Voice IDs are backend-specific and NOT portable. ``speech.voice_id`` applies
+# only to ``speech.tts_backend``; if synthesis falls back to another backend we
+# use that backend's own default rather than passing an unrecognized ID through.
+BACKEND_DEFAULT_VOICE = {
+ "kokoro": "bm_george", # British male
+ "openai_tts": "onyx", # deepest OpenAI preset
+ "cartesia": "", # no safe static default; let Cartesia choose
+}
+
+
+def default_voice_for(backend_id: str) -> str:
+ """Return the fallback voice ID for *backend_id*, or an empty string."""
+ return BACKEND_DEFAULT_VOICE.get(backend_id, "")
+
+
+def get_tts_backend(
+ preferred: str = "",
+ *,
+ attempted: Optional[set[str]] = None,
+) -> Optional[Any]:
+ """Return the first healthy TTS backend, preferring *preferred*.
+
+ ``attempted`` lets a caller carry state across calls so a backend that
+ already failed is not retried; it is mutated in place.
+ """
+ # Import triggers built-in backend registration only when voice output is
+ # actually requested.
+ import openjarvis.speech # noqa: F401
+ from openjarvis.core.registry import TTSRegistry
+
+ seen = attempted if attempted is not None else set()
+
+ for key in dict.fromkeys((preferred, *TTS_BACKEND_ORDER)):
+ if not key or key in seen:
+ continue
+ seen.add(key)
+ if not TTSRegistry.contains(key):
+ continue
+ try:
+ candidate = TTSRegistry.get(key)()
+ if candidate.health():
+ return candidate
+ except Exception:
+ logger.debug("TTS backend %s unavailable", key, exc_info=True)
+ continue
+ return None
+
+
+def voice_preferences(config: "JarvisConfig") -> tuple[str, str, float]:
+ """Resolve ``(tts_backend, voice_id, speed)`` from *config*."""
+ speech = getattr(config, "speech", None)
+ return (
+ getattr(speech, "tts_backend", "kokoro") or "kokoro",
+ getattr(speech, "voice_id", "") or "",
+ float(getattr(speech, "voice_speed", 1.0)),
+ )
+
+
+__all__ = [
+ "BACKEND_DEFAULT_VOICE",
+ "TTS_BACKEND_ORDER",
+ "default_voice_for",
+ "get_tts_backend",
+ "voice_preferences",
+]
diff --git a/tests/server/test_middleware.py b/tests/server/test_middleware.py
index 451cc973..58ac7a73 100644
--- a/tests/server/test_middleware.py
+++ b/tests/server/test_middleware.py
@@ -118,3 +118,23 @@ class TestSecurityHeaders:
assert "access-control-allow-origin" in resp.headers
# Security headers should NOT be present on preflight
assert "X-Frame-Options" not in resp.headers
+
+
+class TestSpeechHeaders:
+ """The security headers must not block the web UI's own speech features."""
+
+ def test_csp_allows_blob_media(self):
+ """Synthesized speech reaches the player as a blob URL."""
+ csp = SECURITY_HEADERS["Content-Security-Policy"]
+ assert "media-src 'self' blob:" in csp
+
+ def test_microphone_allowed_for_same_origin(self):
+ """An empty allowlist would block the chat UI's own voice input."""
+ policy = SECURITY_HEADERS["Permissions-Policy"]
+ assert "microphone=(self)" in policy
+ assert "microphone=()" not in policy
+
+ def test_camera_and_geolocation_stay_blocked(self):
+ policy = SECURITY_HEADERS["Permissions-Policy"]
+ assert "camera=()" in policy
+ assert "geolocation=()" in policy
diff --git a/tests/server/test_speech_routes.py b/tests/server/test_speech_routes.py
index d5390367..6faa1ff6 100644
--- a/tests/server/test_speech_routes.py
+++ b/tests/server/test_speech_routes.py
@@ -1,5 +1,8 @@
"""Tests for speech API endpoints."""
+import asyncio
+import time
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -7,6 +10,7 @@ import pytest
fastapi = pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
+from httpx import ASGITransport, AsyncClient # noqa: E402
from openjarvis.speech._stubs import TranscriptionResult # noqa: E402
@@ -137,3 +141,246 @@ def test_health_no_backend():
assert response.status_code == 200
data = response.json()
assert data["available"] is False
+
+
+# ---------------------------------------------------------------------------
+# Text-to-speech
+# ---------------------------------------------------------------------------
+
+
+def mock_tts_backend():
+ from openjarvis.speech.tts import TTSResult
+
+ backend = MagicMock()
+ backend.backend_id = "mock_tts"
+ backend.health.return_value = True
+ backend.synthesize.return_value = TTSResult(
+ audio=b"RIFFfake-wav-bytes",
+ format="wav",
+ duration_seconds=1.25,
+ voice_id="mock-voice",
+ sample_rate=22050,
+ )
+ return backend
+
+
+def _tts_app(backend, *, config=None):
+ from fastapi import FastAPI
+
+ from openjarvis.server.api_routes import speech_router
+
+ app = FastAPI()
+ app.include_router(speech_router)
+ # Pre-resolve so the route does not go through backend discovery.
+ app.state.tts_backend = backend
+ app.state.tts_resolved = True
+ app.state.config = config
+ return TestClient(app)
+
+
+def _unresolved_tts_app():
+ from fastapi import FastAPI
+
+ from openjarvis.server.api_routes import speech_router
+
+ app = FastAPI()
+ app.include_router(speech_router)
+ app.state.config = SimpleNamespace(
+ speech=SimpleNamespace(
+ tts_backend="kokoro", voice_id="bm_george", voice_speed=1.0
+ )
+ )
+ return app
+
+
+def test_synthesize_returns_wav(mock_tts_backend):
+ client = _tts_app(mock_tts_backend)
+
+ response = client.post("/v1/speech/synthesize", json={"text": "Hallo Welt"})
+
+ assert response.status_code == 200
+ assert response.headers["content-type"] == "audio/wav"
+ assert response.headers["x-sample-rate"] == "22050"
+ assert response.headers["x-voice-id"] == "mock-voice"
+ assert response.content == b"RIFFfake-wav-bytes"
+ assert mock_tts_backend.synthesize.call_args.kwargs["output_format"] == "wav"
+
+
+def test_synthesize_honours_request_overrides(mock_tts_backend):
+ client = _tts_app(mock_tts_backend)
+
+ client.post(
+ "/v1/speech/synthesize",
+ json={"text": "Hallo", "voice_id": "de_DE-thorsten-medium", "speed": 1.5},
+ )
+
+ kwargs = mock_tts_backend.synthesize.call_args.kwargs
+ assert kwargs["voice_id"] == "de_DE-thorsten-medium"
+ assert kwargs["speed"] == 1.5
+
+
+def test_synthesize_rejects_empty_text(mock_tts_backend):
+ client = _tts_app(mock_tts_backend)
+
+ response = client.post("/v1/speech/synthesize", json={"text": " "})
+
+ assert response.status_code == 400
+ mock_tts_backend.synthesize.assert_not_called()
+
+
+def test_synthesize_rejects_oversized_text(mock_tts_backend):
+ from openjarvis.server.api_routes import _MAX_TTS_CHARS
+
+ client = _tts_app(mock_tts_backend)
+
+ response = client.post(
+ "/v1/speech/synthesize", json={"text": "a" * (_MAX_TTS_CHARS + 1)}
+ )
+
+ assert response.status_code == 413
+ mock_tts_backend.synthesize.assert_not_called()
+
+
+def test_synthesize_without_backend_returns_501():
+ client = _tts_app(None)
+
+ response = client.post("/v1/speech/synthesize", json={"text": "Hallo"})
+
+ assert response.status_code == 501
+
+
+def test_synthesize_surfaces_backend_failure(mock_tts_backend):
+ mock_tts_backend.synthesize.side_effect = RuntimeError("voice model missing")
+ client = _tts_app(mock_tts_backend)
+
+ response = client.post("/v1/speech/synthesize", json={"text": "Hallo"})
+
+ assert response.status_code == 500
+ assert response.json()["detail"] == "Speech synthesis failed"
+
+
+def test_synthesize_offloads_backend_work(mock_tts_backend):
+ from openjarvis.speech.tts import TTSResult
+
+ client = _tts_app(mock_tts_backend)
+ expected = TTSResult(audio=b"RIFFoffloaded", format="wav", sample_rate=22050)
+
+ with patch(
+ "openjarvis.server.api_routes.asyncio.to_thread",
+ new_callable=AsyncMock,
+ return_value=expected,
+ ) as to_thread:
+ response = client.post("/v1/speech/synthesize", json={"text": "Hallo"})
+
+ assert response.content == b"RIFFoffloaded"
+ to_thread.assert_awaited_once()
+
+
+def test_tts_health_reports_backend(mock_tts_backend):
+ client = _tts_app(mock_tts_backend)
+
+ data = client.get("/v1/speech/tts/health").json()
+
+ assert data["available"] is True
+ assert data["backend"] == "mock_tts"
+
+
+def test_tts_health_without_backend():
+ client = _tts_app(None)
+
+ data = client.get("/v1/speech/tts/health").json()
+
+ assert data["available"] is False
+ assert "reason" in data
+
+
+def test_tts_health_retries_after_unavailable(mock_tts_backend):
+ app = _unresolved_tts_app()
+ client = TestClient(app)
+
+ with patch(
+ "openjarvis.speech._tts_discovery.get_tts_backend",
+ side_effect=[None, mock_tts_backend],
+ ) as discover:
+ assert client.get("/v1/speech/tts/health").json()["available"] is False
+ assert client.get("/v1/speech/tts/health").json()["available"] is True
+ assert client.get("/v1/speech/tts/health").json()["available"] is True
+
+ assert discover.call_count == 2
+
+
+async def test_concurrent_tts_health_requests_share_resolution(mock_tts_backend):
+ app = _unresolved_tts_app()
+
+ def discover(_preferred):
+ time.sleep(0.05)
+ return mock_tts_backend
+
+ with patch(
+ "openjarvis.speech._tts_discovery.get_tts_backend", side_effect=discover
+ ) as mock_discover:
+ async with AsyncClient(
+ transport=ASGITransport(app=app), base_url="http://testserver"
+ ) as client:
+ responses = await asyncio.gather(
+ *(client.get("/v1/speech/tts/health") for _ in range(3))
+ )
+
+ assert all(response.json()["available"] is True for response in responses)
+ mock_discover.assert_called_once_with("kokoro")
+
+
+async def test_concurrent_unavailable_probes_share_attempt_then_retry(mock_tts_backend):
+ app = _unresolved_tts_app()
+ attempts = 0
+
+ def discover(_preferred):
+ nonlocal attempts
+ attempts += 1
+ time.sleep(0.05)
+ return None if attempts == 1 else mock_tts_backend
+
+ with patch(
+ "openjarvis.speech._tts_discovery.get_tts_backend", side_effect=discover
+ ):
+ async with AsyncClient(
+ transport=ASGITransport(app=app), base_url="http://testserver"
+ ) as client:
+ responses = await asyncio.gather(
+ *(client.get("/v1/speech/tts/health") for _ in range(3))
+ )
+ assert all(response.json()["available"] is False for response in responses)
+ assert attempts == 1
+ assert (await client.get("/v1/speech/tts/health")).json()["available"]
+
+ assert attempts == 2
+
+
+def test_invalid_synthesis_does_not_load_tts_backend():
+ client = TestClient(_unresolved_tts_app())
+
+ with patch("openjarvis.speech._tts_discovery.get_tts_backend") as discover:
+ response = client.post("/v1/speech/synthesize", json={"text": " "})
+
+ assert response.status_code == 400
+ discover.assert_not_called()
+
+
+def test_voice_id_not_reused_across_backends(mock_tts_backend):
+ """A configured voice belongs to its backend; a fallback uses its own."""
+ from types import SimpleNamespace
+
+ config = SimpleNamespace(
+ speech=SimpleNamespace(
+ tts_backend="kokoro", voice_id="bm_george", voice_speed=1.0
+ )
+ )
+ client = _tts_app(mock_tts_backend, config=config)
+
+ client.post("/v1/speech/synthesize", json={"text": "Hallo"})
+
+ # Backend is mock_tts, not the configured kokoro, so bm_george must not leak.
+ assert mock_tts_backend.synthesize.call_args.kwargs["voice_id"] != "bm_george"