Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(frontend/copilot): read aloud falls back to our own TTS when the browser has no voices (#14354)

Reinier van der Leerff309b92a7b8
diff --git a/autogpt_platform/backend/backend/api/features/chat/speech_test.py b/autogpt_platform/backend/backend/api/features/chat/speech_test.py
index 75954f45fc..c64825b23c 100644
--- a/autogpt_platform/backend/backend/api/features/chat/speech_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/speech_test.py
@@ -204,6 +204,18 @@ def test_speech_503s_without_a_configured_key(
record_usage.assert_not_awaited()
+def test_speech_bounds_a_hung_upstream(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The client's own 600 s default, across its two retries, is half an hour of
+ # silence on the path voice mode chunks to speak within a second.
+ monkeypatch.setattr(speech_module.settings.secrets, "openai_internal_api_key", "")
+ monkeypatch.setattr(speech_module.settings.secrets, "openai_api_key", "sk-test-key")
+
+ timeout = speech_module._speech_client().timeout
+
+ assert timeout == speech_module.SPEECH_TIMEOUT
+ assert (timeout.read, timeout.connect) == (30.0, 5.0)
+
+
@pytest.fixture(autouse=True)
def setup_app_auth(mock_jwt_user, mocker: pytest_mock.MockerFixture):
from autogpt_libs.auth.dependencies import get_request_context
diff --git a/autogpt_platform/backend/backend/copilot/speech.py b/autogpt_platform/backend/backend/copilot/speech.py
index 0f196092d9..a9f9ac9d59 100644
--- a/autogpt_platform/backend/backend/copilot/speech.py
+++ b/autogpt_platform/backend/backend/copilot/speech.py
@@ -7,6 +7,7 @@ in-plan usage rather than a free side channel.
import logging
+import httpx
from openai import AsyncOpenAI
from backend.copilot.config import ChatConfig
@@ -22,6 +23,10 @@ settings = Settings()
# boundaries well below this, so hitting it means a malformed request.
MAX_SPEECH_CHARS = 4096
+# One chunk is a sentence, not a generation, so the client's 600 s default bounds
+# nothing a listener is still waiting through. Connect fast-fails as elsewhere.
+SPEECH_TIMEOUT = httpx.Timeout(30.0, connect=5.0)
+
ALLOWED_VOICES = frozenset(
{
"alloy",
@@ -137,4 +142,4 @@ def _speech_client() -> AsyncOpenAI:
)
if not api_key:
raise SpeechUnavailable("no OpenAI API key configured")
- return AsyncOpenAI(api_key=api_key)
+ return AsyncOpenAI(api_key=api_key, timeout=SPEECH_TIMEOUT)
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx
index a82f214300..0262047e91 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/AssistantMessageActions.tsx
@@ -89,7 +89,7 @@ export function AssistantMessageActions({
<Icon icon={ThumbsDownIcon} size={16} />
</MessageAction>
- <TTSButton text={text} />
+ <TTSButton text={text} sessionID={sessionID} />
</MessageActions>
{showFeedbackModal && (
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/TTSButton.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/TTSButton.tsx
index d9c467d7d3..95c1bb9429 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/TTSButton.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/TTSButton.tsx
@@ -1,23 +1,19 @@
"use client";
import { MessageAction } from "@/components/ai-elements/message";
-import { useTextToSpeech } from "@/components/contextual/Chat/components/ChatMessage/useTextToSpeech";
-import { stripMarkdownForSpeech } from "../../../voice/stripMarkdownForSpeech";
-import { useMemo } from "react";
import { StopIcon, VolumeHighIcon } from "@hugeicons/core-free-icons";
import { Icon } from "@/components/atoms/Icon/Icon";
+import { useTTSButton } from "./useTTSButton";
interface Props {
text: string;
+ sessionID: string | null;
}
-export function TTSButton({ text }: Props) {
- const cleanText = useMemo(() => stripMarkdownForSpeech(text), [text]);
- const { status, isSupported, toggle } = useTextToSpeech(cleanText);
+export function TTSButton({ text, sessionID }: Props) {
+ const { canSpeak, isPlaying, toggle } = useTTSButton({ text, sessionID });
- if (!isSupported || !cleanText) return null;
-
- const isPlaying = status === "playing";
+ if (!canSpeak) return null;
return (
<MessageAction
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/__tests__/TTSButton.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/__tests__/TTSButton.test.tsx
new file mode 100644
index 0000000000..b996416a9a
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/__tests__/TTSButton.test.tsx
@@ -0,0 +1,191 @@
+import {
+ act,
+ cleanup,
+ render as rtlRender,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+// Hoisted: `vi.mock` factories are lifted above every top-level binding.
+const { synthesizeSpeech, useGetFlag } = vi.hoisted(() => ({
+ synthesizeSpeech: vi.fn(
+ async (text: string, sessionID: string | null) =>
+ new Blob([`${sessionID}:${text}`]),
+ ),
+ useGetFlag: vi.fn(() => true),
+}));
+
+vi.mock("../../../../voice/speechApi", () => ({
+ synthesizeSpeech,
+}));
+
+vi.mock("@/services/feature-flags/use-get-flag", () => ({
+ Flag: { COPILOT_VOICE_MODE: "copilot-voice-mode" },
+ useGetFlag: () => useGetFlag(),
+}));
+
+vi.mock("@/components/molecules/Toast/use-toast", () => ({
+ useToast: () => ({ toast: vi.fn() }),
+}));
+
+import { TooltipProvider } from "@/components/atoms/Tooltip/BaseTooltip";
+import { TTSButton } from "../TTSButton";
+
+function render(ui: React.ReactElement) {
+ return rtlRender(<TooltipProvider>{ui}</TooltipProvider>);
+}
+
+/** speechPlayer keeps one module-level <audio>, so these outlive a single test. */
+let autoEndPlayback = true;
+const audioPause = vi.fn();
+
+describe("TTSButton", () => {
+ beforeEach(() => {
+ autoEndPlayback = true;
+ audioPause.mockClear();
+ stubSpeechSynthesis([]);
+ stubAudio();
+ global.URL.createObjectURL = vi.fn(() => "blob:chunk");
+ global.URL.revokeObjectURL = vi.fn();
+ useGetFlag.mockReturnValue(true);
+ synthesizeSpeech.mockClear();
+ });
+
+ afterEach(() => {
+ cleanup();
+ vi.unstubAllGlobals();
+ });
+
+ it("speaks through the browser when it has voices, without spending on synthesis", async () => {
+ const { speak } = stubSpeechSynthesis([
+ { name: "Samantha", lang: "en-US" },
+ ]);
+ render(<TTSButton text="A local voice reads this." sessionID="s-1" />);
+
+ await userEvent.click(await screen.findByRole("button"));
+
+ expect(speak).toHaveBeenCalledTimes(1);
+ expect(synthesizeSpeech).not.toHaveBeenCalled();
+ });
+
+ it("falls back to our own synthesis when the browser has no voices", async () => {
+ render(<TTSButton text="No local voice reads this." sessionID="s-1" />);
+
+ await userEvent.click(await screen.findByRole("button"));
+
+ await waitFor(() => expect(synthesizeSpeech).toHaveBeenCalled());
+ expect(synthesizeSpeech.mock.calls[0]).toEqual([
+ "No local voice reads this.",
+ "s-1",
+ ]);
+ });
+
+ it("splits a reply too long for one request into several", async () => {
+ const long = "This sentence is here to be spoken aloud. ".repeat(30);
+ render(<TTSButton text={long} sessionID={null} />);
+
+ await userEvent.click(await screen.findByRole("button"));
+
+ await waitFor(() => expect(synthesizeSpeech).toHaveBeenCalled());
+ const chunks = synthesizeSpeech.mock.calls.map((call) => call[0]);
+ expect(chunks.length).toBeGreaterThan(1);
+ chunks.forEach((chunk) => expect(chunk.length).toBeLessThanOrEqual(4096));
+ });
+
+ it("attributes the cost to the session the message belongs to", async () => {
+ render(<TTSButton text="Bill this to the session." sessionID="s-42" />);
+
+ await userEvent.click(await screen.findByRole("button"));
+
+ await waitFor(() => expect(synthesizeSpeech).toHaveBeenCalled());
+ expect(synthesizeSpeech.mock.calls[0][1]).toBe("s-42");
+ });
+
+ it("hides itself when it has neither voices nor the flag that lets it synthesise", () => {
+ useGetFlag.mockReturnValue(false);
+ render(<TTSButton text="Nothing can speak this." sessionID="s-1" />);
+
+ expect(screen.queryByRole("button")).toBeNull();
+ });
+
+ it("still offers browser speech when the flag is off but voices exist", async () => {
+ useGetFlag.mockReturnValue(false);
+ const { speak } = stubSpeechSynthesis([
+ { name: "Samantha", lang: "en-US" },
+ ]);
+ render(<TTSButton text="A local voice reads this." sessionID="s-1" />);
+
+ await userEvent.click(await screen.findByRole("button"));
+
+ expect(speak).toHaveBeenCalledTimes(1);
+ expect(synthesizeSpeech).not.toHaveBeenCalled();
+ });
+
+ it("keeps Stop on our own audio when voices arrive mid-playback", async () => {
+ // speech-dispatcher starting mid-reply flips `hasVoices`, which would
+ // otherwise hand Stop to the browser path and strand our audio playing.
+ autoEndPlayback = false;
+ const { speak, arriveVoices } = stubSpeechSynthesis([]);
+ render(<TTSButton text="No local voice reads this." sessionID="s-1" />);
+
+ await userEvent.click(await screen.findByRole("button"));
+ await waitFor(() => expect(synthesizeSpeech).toHaveBeenCalled());
+ act(() => arriveVoices([{ name: "Samantha", lang: "en-US" }]));
+
+ await userEvent.click(screen.getByRole("button"));
+
+ // Stop has to actually stop our audio — `speak` alone stays uncalled when
+ // the click does nothing at all, which is the same dead control.
+ expect(audioPause).toHaveBeenCalled();
+ expect(speak).not.toHaveBeenCalled();
+ });
+});
+
+function stubSpeechSynthesis(voices: Partial<SpeechSynthesisVoice>[]) {
+ const speak = vi.fn();
+ let current = voices;
+ let onVoicesChanged = () => {};
+ vi.stubGlobal(
+ "SpeechSynthesisUtterance",
+ class {
+ constructor(public text: string) {}
+ },
+ );
+ vi.stubGlobal("speechSynthesis", {
+ getVoices: () => current as SpeechSynthesisVoice[],
+ speak,
+ cancel: vi.fn(),
+ pause: vi.fn(),
+ resume: vi.fn(),
+ addEventListener: vi.fn((event: string, handler: () => void) => {
+ if (event === "voiceschanged") onVoicesChanged = handler;
+ }),
+ removeEventListener: vi.fn(),
+ });
+ return {
+ speak,
+ arriveVoices(later: Partial<SpeechSynthesisVoice>[]) {
+ current = later;
+ onVoicesChanged();
+ },
+ };
+}
+
+function stubAudio() {
+ vi.stubGlobal(
+ "Audio",
+ class {
+ src = "";
+ preload = "";
+ play = vi.fn(async () => undefined);
+ pause = audioPause;
+ addEventListener = vi.fn(
+ (event: string, handler: () => void) =>
+ autoEndPlayback && event === "ended" && setTimeout(handler, 0),
+ );
+ removeEventListener = vi.fn();
+ },
+ );
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/useTTSButton.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/useTTSButton.ts
new file mode 100644
index 0000000000..0f847cb198
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/useTTSButton.ts
@@ -0,0 +1,103 @@
+"use client";
+
+import { useEffect, useMemo, useRef, useState } from "react";
+
+import { useTextToSpeech } from "@/components/contextual/Chat/components/ChatMessage/useTextToSpeech";
+import { useToast } from "@/components/molecules/Toast/use-toast";
+import { Flag, useGetFlag } from "@/services/feature-flags/use-get-flag";
+
+import { synthesizeSpeech } from "../../../voice/speechApi";
+import { takeSpeakableChunks } from "../../../voice/speechChunker";
+import {
+ createSpeechPlayer,
+ type SpeechPlayer,
+} from "../../../voice/speechPlayer";
+import { stripMarkdownForSpeech } from "../../../voice/stripMarkdownForSpeech";
+
+interface Args {
+ text: string;
+ sessionID: string | null;
+}
+
+export function useTTSButton({ text, sessionID }: Args) {
+ const cleanText = useMemo(() => stripMarkdownForSpeech(text), [text]);
+ const browser = useTextToSpeech(cleanText);
+ const server = useServerSpeech(cleanText, sessionID);
+
+ // The speech route 404s when voice mode is off for the user, so without the
+ // flag there is nothing to fall back to and the button must not offer one.
+ const canFallBack = useGetFlag(Flag.COPILOT_VOICE_MODE);
+ // `hasVoices` flips reactively on `voiceschanged`; a flip mid-playback would
+ // move Stop to the other path and strand the audio that is already speaking.
+ const speaking = server.isPlaying || browser.status === "playing";
+ const viaServer = speaking ? server.isPlaying : !browser.hasVoices;
+
+ return {
+ canSpeak: Boolean(cleanText) && (browser.hasVoices || canFallBack),
+ isPlaying: viaServer ? server.isPlaying : browser.status === "playing",
+ toggle: viaServer ? server.toggle : browser.toggle,
+ };
+}
+
+function useServerSpeech(text: string, sessionID: string | null) {
+ const { toast } = useToast();
+ const [isPlaying, setIsPlaying] = useState(false);
+ const playerRef = useRef<SpeechPlayer | null>(null);
+ const reportedRef = useRef(false);
+ const sessionRef = useRef(sessionID);
+ sessionRef.current = sessionID;
+
+ useEffect(() => {
+ playerRef.current?.stop();
+ setIsPlaying(false);
+ }, [text]);
+
+ useEffect(() => () => playerRef.current?.destroy(), []);
+
+ return { isPlaying, toggle };
+
+ function toggle() {
+ if (isPlaying) {
+ playerRef.current?.stop();
+ setIsPlaying(false);
+ return;
+ }
+
+ const player = ensurePlayer();
+ // Must happen inside the click: that is the only gesture granting the
+ // shared <audio> element autoplay for the chunks that arrive later.
+ player.unlock();
+ reportedRef.current = false;
+ setIsPlaying(true);
+ // One request is capped at 4096 characters, which a long reply exceeds.
+ takeSpeakableChunks(text, true).chunks.forEach((chunk) =>
+ player.enqueue(chunk),
+ );
+ }
+
+ function ensurePlayer(): SpeechPlayer {
+ if (!playerRef.current) {
+ playerRef.current = createSpeechPlayer({
+ synthesize: (chunk) => synthesizeSpeech(chunk, sessionRef.current),
+ onIdle: () => setIsPlaying(false),
+ onError: report,
+ });
+ }
+ return playerRef.current;
+ }
+
+ // Every remaining chunk of a failed reply fails the same way; say it once.
+ function report(error: unknown) {
+ if (reportedRef.current) return;
+ reportedRef.current = true;
+ playerRef.current?.stop();
+ setIsPlaying(false);
+ console.error("[Read aloud]", error);
+ toast({
+ title: "Read aloud",
+ description:
+ error instanceof Error ? error.message : "Something went wrong.",
+ variant: "destructive",
+ });
+ }
+}
diff --git a/autogpt_platform/frontend/src/components/contextual/Chat/components/ChatMessage/useTextToSpeech.ts b/autogpt_platform/frontend/src/components/contextual/Chat/components/ChatMessage/useTextToSpeech.ts
index ebfdb84608..00b81b13da 100644
--- a/autogpt_platform/frontend/src/components/contextual/Chat/components/ChatMessage/useTextToSpeech.ts
+++ b/autogpt_platform/frontend/src/components/contextual/Chat/components/ChatMessage/useTextToSpeech.ts
@@ -89,35 +89,28 @@ function pickBestVoice(
export function useTextToSpeech(text: string) {
const [status, setStatus] = useState<TTSStatus>("idle");
- const [isSupported, setIsSupported] = useState(false);
+ // The API exists with zero voices (Linux without a running speech-dispatcher),
+ // where `speak` is a silent no-op — so the voice list, not the API, is the
+ // test for whether this can speak. State, not a ref: voices load async.
+ const [voices, setVoices] = useState<SpeechSynthesisVoice[]>([]);
const utteranceRef = useRef<SpeechSynthesisUtterance | null>(null);
- const cachedVoicesRef = useRef<SpeechSynthesisVoice[]>([]);
useEffect(() => {
- const supported = "speechSynthesis" in window;
- setIsSupported(supported);
-
- if (supported) {
- // Populate cache immediately (may already be available)
- cachedVoicesRef.current = window.speechSynthesis.getVoices();
-
- // Listen for voiceschanged to populate cache when voices load async
- function onVoicesChanged() {
- cachedVoicesRef.current = window.speechSynthesis.getVoices();
- }
- window.speechSynthesis.addEventListener("voiceschanged", onVoicesChanged);
-
- return () => {
- window.speechSynthesis.removeEventListener(
- "voiceschanged",
- onVoicesChanged,
- );
- window.speechSynthesis.cancel();
- };
+ if (!("speechSynthesis" in window)) return;
+
+ setVoices(window.speechSynthesis.getVoices());
+
+ function onVoicesChanged() {
+ setVoices(window.speechSynthesis.getVoices());
}
+ window.speechSynthesis.addEventListener("voiceschanged", onVoicesChanged);
return () => {
- window.speechSynthesis?.cancel();
+ window.speechSynthesis.removeEventListener(
+ "voiceschanged",
+ onVoicesChanged,
+ );
+ window.speechSynthesis.cancel();
};
}, []);
@@ -142,7 +135,7 @@ export function useTextToSpeech(text: string) {
const utterance = new SpeechSynthesisUtterance(text);
- const voice = pickBestVoice(cachedVoicesRef.current);
+ const voice = pickBestVoice(voices);
if (voice) utterance.voice = voice;
utteranceRef.current = utterance;
@@ -184,7 +177,7 @@ export function useTextToSpeech(text: string) {
return {
status,
- isSupported,
+ hasVoices: voices.length > 0,
play,
pause,
stop,