Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔊 fix: Autoplay Latest Message for Browser TTS and short responses (#15347)

Danny Avila36f129089b31
diff --git a/client/src/components/Chat/Input/AutoPlayAudio.tsx b/client/src/components/Chat/Input/AutoPlayAudio.tsx
new file mode 100644
index 0000000000..7cc5e4a66b
--- /dev/null
+++ b/client/src/components/Chat/Input/AutoPlayAudio.tsx
@@ -0,0 +1,34 @@
+import { memo } from 'react';
+import { useRecoilValue } from 'recoil';
+import BrowserAudio from './BrowserAudio';
+import StreamAudio from './StreamAudio';
+import { TTSEndpoints } from '~/common';
+import store from '~/store';
+
+/**
+ * Mount point for "Autoplay Latest Message". Playback has to honor the selected TTS engine
+ * the same way the message speaker button does, otherwise the Browser engine silently falls
+ * through to the external endpoint and never reaches the Web Speech API.
+ */
+function AutoPlayAudio({ index = 0 }) {
+ const engineTTS = useRecoilValue<string>(store.engineTTS);
+ const speechSettingsInitialized = useRecoilValue(store.speechSettingsInitialized);
+
+ if (!speechSettingsInitialized) {
+ return null;
+ }
+
+ const AutoPlayComponents = {
+ [TTSEndpoints.browser]: BrowserAudio,
+ [TTSEndpoints.external]: StreamAudio,
+ };
+
+ const SelectedAutoPlay = AutoPlayComponents[engineTTS];
+ if (!SelectedAutoPlay) {
+ return null;
+ }
+
+ return <SelectedAutoPlay index={index} />;
+}
+
+export default memo(AutoPlayAudio);
diff --git a/client/src/components/Chat/Input/BrowserAudio.tsx b/client/src/components/Chat/Input/BrowserAudio.tsx
new file mode 100644
index 0000000000..262310f7b2
--- /dev/null
+++ b/client/src/components/Chat/Input/BrowserAudio.tsx
@@ -0,0 +1,57 @@
+import { useEffect } from 'react';
+import { useSetRecoilState } from 'recoil';
+import { useParams } from 'react-router-dom';
+import { parseTextParts } from 'librechat-data-provider';
+import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';
+import useAutoplayTrigger from '~/hooks/Audio/useAutoplayTrigger';
+import { logger } from '~/utils';
+import store from '~/store';
+
+/**
+ * Autoplay driver for the Browser TTS engine. `StreamAudio` covers the external engine by
+ * streaming server audio into the global `<audio>` element; the Web Speech API has no such
+ * element, so the utterance is spoken directly off the shared autoplay gate.
+ */
+export default function BrowserAudio({ index = 0 }) {
+ const { conversationId: paramId } = useParams();
+ const setIsSpeaking = useSetRecoilState(store.globalAudioPlayingFamily(index));
+ const setAudioRunId = useSetRecoilState(store.audioRunFamily(index));
+
+ const { shouldPlay, activeRunId, latestMessage } = useAutoplayTrigger(index);
+ const { generateSpeechLocal, cancelSpeechLocal } = useTextToSpeechBrowser({ setIsSpeaking });
+
+ /**
+ * Leaving a conversation (or unmounting) stops whatever is still being spoken. React runs
+ * every cleanup before any effect body, so the utterance the effect below is about to start
+ * survives the `/c/new` -> `/c/:id` navigation that lands in the same commit as finalization.
+ */
+ useEffect(() => {
+ return () => cancelSpeechLocal();
+ // We only want the effect to run when the paramId changes
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [paramId]);
+
+ useEffect(() => {
+ if (!shouldPlay || activeRunId == null || latestMessage == null) {
+ return;
+ }
+
+ const text =
+ Array.isArray(latestMessage.content) && latestMessage.content.length > 0
+ ? parseTextParts(latestMessage.content)
+ : (latestMessage.text ?? '');
+
+ if (!text) {
+ return;
+ }
+
+ logger.log('BrowserAudio.tsx - speaking message:', latestMessage.messageId);
+ /** Only claim the run once an utterance was queued, so a not-yet-loaded voice list
+ * retries on the next render instead of silently swallowing the playback. */
+ if (generateSpeechLocal(text)) {
+ setAudioRunId(activeRunId);
+ }
+ }, [shouldPlay, activeRunId, latestMessage, setAudioRunId, generateSpeechLocal]);
+
+ return null;
+}
diff --git a/client/src/components/Chat/Input/ChatForm.tsx b/client/src/components/Chat/Input/ChatForm.tsx
index 0bfe3f7416..0a1266216a 100644
--- a/client/src/components/Chat/Input/ChatForm.tsx
+++ b/client/src/components/Chat/Input/ChatForm.tsx
@@ -51,9 +51,9 @@ import TextareaHeader from './TextareaHeader';
import PromptsCommand from './PromptsCommand';
import SkillsCommand from './SkillsCommand';
import AudioRecorder from './AudioRecorder';
+import AutoPlayAudio from './AutoPlayAudio';
import CollapseChat from './CollapseChat';
import QuoteButton from './QuoteButton';
-import StreamAudio from './StreamAudio';
import TokenUsage from './TokenUsage';
import StopButton from './StopButton';
import SendButton from './SendButton';
@@ -769,7 +769,7 @@ const ChatForm = memo(function ChatForm({
)}
</div>
</div>
- {TextToSpeech && automaticPlayback && <StreamAudio index={index} />}
+ {TextToSpeech && automaticPlayback && <AutoPlayAudio index={index} />}
</div>
</div>
</div>
diff --git a/client/src/components/Chat/Input/StreamAudio.tsx b/client/src/components/Chat/Input/StreamAudio.tsx
index 09011b27ab..d7e788da46 100644
--- a/client/src/components/Chat/Input/StreamAudio.tsx
+++ b/client/src/components/Chat/Input/StreamAudio.tsx
@@ -4,11 +4,15 @@ import { QueryKeys } from 'librechat-data-provider';
import { useQueryClient } from '@tanstack/react-query';
import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';
import type { TMessage } from 'librechat-data-provider';
-import { useCustomAudioRef, MediaSourceAppender, usePauseGlobalAudio } from '~/hooks/Audio';
-import { useLatestMessage } from '~/hooks/Messages/useLatestMessage';
-import { getLatestText, logger } from '~/utils';
+import {
+ useCustomAudioRef,
+ useAutoplayTrigger,
+ MediaSourceAppender,
+ usePauseGlobalAudio,
+} from '~/hooks/Audio';
import { useAuthContext } from '~/hooks';
import { globalAudioId } from '~/common';
+import { logger } from '~/utils';
import store from '~/store';
function timeoutPromise(ms: number, message?: string) {
@@ -27,15 +31,13 @@ export default function StreamAudio({ index = 0 }) {
const playbackRate = useRecoilValue(store.playbackRate);
const voice = useRecoilValue(store.voice);
- const activeRunId = useRecoilValue(store.activeRunFamily(index));
const automaticPlayback = useRecoilValue(store.automaticPlayback);
- const isSubmitting = useRecoilValue(store.isSubmittingFamily(index));
- const latestMessage = useLatestMessage(index);
const setIsPlaying = useSetRecoilState(store.globalAudioPlayingFamily(index));
- const [audioRunId, setAudioRunId] = useRecoilState(store.audioRunFamily(index));
+ const setAudioRunId = useSetRecoilState(store.audioRunFamily(index));
const [isFetching, setIsFetching] = useRecoilState(store.globalAudioFetchingFamily(index));
const [globalAudioURL, setGlobalAudioURL] = useRecoilState(store.globalAudioURLFamily(index));
+ const { shouldPlay, activeRunId, latestMessage } = useAutoplayTrigger(index);
const { audioRef } = useCustomAudioRef({ setIsPlaying });
const { pauseGlobalAudio } = usePauseGlobalAudio();
@@ -49,28 +51,13 @@ export default function StreamAudio({ index = 0 }) {
);
useEffect(() => {
- const latestText = getLatestText(latestMessage);
-
- const shouldFetch = !!(
- token != null &&
- automaticPlayback &&
- !isSubmitting &&
- latestMessage &&
- !latestMessage.isCreatedByUser &&
- latestText &&
- latestMessage.messageId &&
- !latestMessage.messageId.includes('_') &&
- !isFetching &&
- activeRunId != null &&
- activeRunId !== audioRunId
- );
-
- if (!shouldFetch) {
+ if (!(token != null && automaticPlayback && shouldPlay && !isFetching)) {
return;
}
async function fetchAudio() {
setIsFetching(true);
+ let mediaSource: MediaSourceAppender | undefined;
try {
if (audioRef.current) {
@@ -112,12 +99,15 @@ export default function StreamAudio({ index = 0 }) {
const type = 'audio/mpeg';
const browserSupportsType =
typeof MediaSource !== 'undefined' && MediaSource.isTypeSupported(type);
- let mediaSource: MediaSourceAppender | undefined;
if (browserSupportsType) {
mediaSource = new MediaSourceAppender(type);
setGlobalAudioURL(mediaSource.mediaSourceUrl);
}
+ /** Browsers without MSE support for the type (Safari, for one) can only be handed the
+ * audio once it is fully read, so that path always buffers — caching opts MSE in too. */
+ const shouldBufferChunks = cacheTTS || !browserSupportsType;
+
let done = false;
const chunks: ArrayBuffer[] = [];
@@ -128,7 +118,7 @@ export default function StreamAudio({ index = 0 }) {
timeoutPromise(maxPromiseTime, promiseTimeoutMessage),
])) as ReadableStreamReadResult<ArrayBuffer>;
- if (cacheTTS && value) {
+ if (shouldBufferChunks && value) {
chunks.push(value);
}
if (value && mediaSource) {
@@ -137,39 +127,37 @@ export default function StreamAudio({ index = 0 }) {
done = readerDone;
}
+ mediaSource?.close();
+
if (chunks.length) {
- logger.log('Adding audio to cache');
- const latestMessages = getMessages() ?? [];
- const targetMessage = latestMessages.find(
- (msg) => msg.messageId === latestMessage?.messageId,
- );
- cacheKey = targetMessage?.text ?? '';
- if (!cacheKey) {
- throw new Error('Cache key not found');
- }
const audioBlob = new Blob(chunks, { type });
- const cachedResponse = new Response(audioBlob);
- await cache.put(cacheKey, cachedResponse);
if (!browserSupportsType) {
- const unconsumedResponse = await cache.match(cacheKey);
- if (!unconsumedResponse) {
- throw new Error('Failed to fetch audio from cache');
+ setGlobalAudioURL(URL.createObjectURL(audioBlob));
+ }
+ if (cacheTTS) {
+ const latestMessages = getMessages() ?? [];
+ const targetMessage = latestMessages.find(
+ (msg) => msg.messageId === latestMessage?.messageId,
+ );
+ cacheKey = targetMessage?.text ?? '';
+ if (!cacheKey) {
+ logger.warn('Cache key not found, skipping audio cache');
+ } else {
+ logger.log('Adding audio to cache');
+ await cache.put(cacheKey, new Response(audioBlob));
}
- const audioBlob = await unconsumedResponse.blob();
- const blobUrl = URL.createObjectURL(audioBlob);
- setGlobalAudioURL(blobUrl);
}
- setIsFetching(false);
}
logger.log('Audio stream reading ended');
} catch (error) {
- if (error?.['message'] !== promiseTimeoutMessage) {
+ if (error?.['message'] === promiseTimeoutMessage) {
logger.log(promiseTimeoutMessage);
+ /** Let whatever was already appended finish playing instead of stalling forever */
+ mediaSource?.close();
return;
}
logger.error('Error fetching audio:', error);
- setIsFetching(false);
setGlobalAudioURL(null);
} finally {
setIsFetching(false);
@@ -183,11 +171,10 @@ export default function StreamAudio({ index = 0 }) {
setAudioRunId,
setIsFetching,
latestMessage,
- isSubmitting,
activeRunId,
getMessages,
+ shouldPlay,
isFetching,
- audioRunId,
cacheTTS,
audioRef,
voice,
diff --git a/client/src/components/Chat/Input/__tests__/AutoPlayAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/AutoPlayAudio.spec.tsx
new file mode 100644
index 0000000000..252d9a28de
--- /dev/null
+++ b/client/src/components/Chat/Input/__tests__/AutoPlayAudio.spec.tsx
@@ -0,0 +1,56 @@
+import React from 'react';
+import { RecoilRoot } from 'recoil';
+import { render, screen } from '@testing-library/react';
+import AutoPlayAudio from '../AutoPlayAudio';
+import store from '~/store';
+
+jest.mock('../BrowserAudio', () => () => <div data-testid="browser-autoplay" />);
+jest.mock('../StreamAudio', () => () => <div data-testid="external-autoplay" />);
+
+const renderAutoPlayAudio = ({
+ engineTTS,
+ initialized = true,
+}: {
+ engineTTS: string;
+ initialized?: boolean;
+}) =>
+ render(
+ <RecoilRoot
+ initializeState={({ set }) => {
+ set(store.engineTTS, engineTTS);
+ set(store.speechSettingsInitialized, initialized);
+ }}
+ >
+ <AutoPlayAudio index={0} />
+ </RecoilRoot>,
+ );
+
+describe('AutoPlayAudio engine selection', () => {
+ it('drives autoplay through the Web Speech API for the browser engine', () => {
+ renderAutoPlayAudio({ engineTTS: 'browser' });
+
+ expect(screen.getByTestId('browser-autoplay')).toBeInTheDocument();
+ expect(screen.queryByTestId('external-autoplay')).not.toBeInTheDocument();
+ });
+
+ it('drives autoplay through the server stream for the external engine', () => {
+ renderAutoPlayAudio({ engineTTS: 'external' });
+
+ expect(screen.getByTestId('external-autoplay')).toBeInTheDocument();
+ expect(screen.queryByTestId('browser-autoplay')).not.toBeInTheDocument();
+ });
+
+ it('mounts nothing before speech settings initialize', () => {
+ renderAutoPlayAudio({ engineTTS: 'browser', initialized: false });
+
+ expect(screen.queryByTestId('browser-autoplay')).not.toBeInTheDocument();
+ expect(screen.queryByTestId('external-autoplay')).not.toBeInTheDocument();
+ });
+
+ it('mounts nothing for an unknown engine', () => {
+ renderAutoPlayAudio({ engineTTS: 'unsupported' });
+
+ expect(screen.queryByTestId('browser-autoplay')).not.toBeInTheDocument();
+ expect(screen.queryByTestId('external-autoplay')).not.toBeInTheDocument();
+ });
+});
diff --git a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx
new file mode 100644
index 0000000000..3dcb631a07
--- /dev/null
+++ b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx
@@ -0,0 +1,149 @@
+import React from 'react';
+import { RecoilRoot } from 'recoil';
+import { render, waitFor } from '@testing-library/react';
+import { Constants, QueryKeys } from 'librechat-data-provider';
+import { MemoryRouter, Routes, Route } from 'react-router-dom';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import type { TMessage, TConversation } from 'librechat-data-provider';
+import BrowserAudio from '../BrowserAudio';
+import store from '~/store';
+
+const conversationId = 'convo-1';
+const voiceName = 'Test Voice';
+const responseText = 'The capital of Germany is Berlin.';
+
+const spoken: string[] = [];
+let cancelCount = 0;
+
+class FakeSpeechSynthesisUtterance {
+ public voice: SpeechSynthesisVoice | null = null;
+ public onend: (() => void) | null = null;
+ public onerror: ((event: { error: string }) => void) | null = null;
+ constructor(public text: string) {}
+}
+
+const fakeVoice = { name: voiceName, localService: true } as SpeechSynthesisVoice;
+
+const speechSynthesis = {
+ getVoices: () => [fakeVoice],
+ addEventListener: () => undefined,
+ speak: (utterance: FakeSpeechSynthesisUtterance) => {
+ spoken.push(utterance.text);
+ },
+ cancel: () => {
+ cancelCount += 1;
+ },
+};
+
+const assistantMessage = {
+ messageId: 'assistant-1',
+ conversationId,
+ parentMessageId: Constants.NO_PARENT,
+ isCreatedByUser: false,
+ text: responseText,
+} as TMessage;
+
+const renderBrowserAudio = ({
+ activeRunId = 'run-1',
+ audioRunId = null,
+ isSubmitting = false,
+ messages = [assistantMessage],
+}: {
+ activeRunId?: string | null;
+ audioRunId?: string | null;
+ isSubmitting?: boolean;
+ messages?: TMessage[];
+} = {}) => {
+ const queryClient = new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+ queryClient.setQueryData([QueryKeys.messages, conversationId], messages);
+
+ return render(
+ <QueryClientProvider client={queryClient}>
+ <MemoryRouter initialEntries={[`/c/${conversationId}`]}>
+ <RecoilRoot
+ initializeState={({ set }) => {
+ set(store.voice, voiceName);
+ set(store.conversationByIndex(0), { conversationId } as TConversation);
+ set(store.activeRunFamily(0), activeRunId);
+ set(store.audioRunFamily(0), audioRunId);
+ set(store.isSubmittingFamily(0), isSubmitting);
+ }}
+ >
+ <Routes>
+ <Route path="/c/:conversationId" element={<BrowserAudio index={0} />} />
+ </Routes>
+ </RecoilRoot>
+ </MemoryRouter>
+ </QueryClientProvider>,
+ );
+};
+
+/** Lets every effect settle so a missing utterance is a real absence, not a pending render */
+const settle = () => waitFor(() => expect(cancelCount).toBeGreaterThanOrEqual(0));
+
+describe('BrowserAudio autoplay', () => {
+ beforeAll(() => {
+ Object.defineProperty(window, 'speechSynthesis', {
+ writable: true,
+ configurable: true,
+ value: speechSynthesis,
+ });
+ Object.defineProperty(global, 'SpeechSynthesisUtterance', {
+ writable: true,
+ configurable: true,
+ value: FakeSpeechSynthesisUtterance,
+ });
+ });
+
+ beforeEach(() => {
+ spoken.length = 0;
+ cancelCount = 0;
+ });
+
+ it('speaks the finalized assistant message through speech synthesis', async () => {
+ renderBrowserAudio();
+
+ await waitFor(() => expect(spoken).toEqual([responseText]));
+ });
+
+ it('does not speak while the run is still submitting', async () => {
+ renderBrowserAudio({ isSubmitting: true });
+ await settle();
+
+ expect(spoken).toEqual([]);
+ });
+
+ it('does not speak a run that was already played', async () => {
+ renderBrowserAudio({ activeRunId: 'run-1', audioRunId: 'run-1' });
+ await settle();
+
+ expect(spoken).toEqual([]);
+ });
+
+ it('does not speak a message that is still streaming', async () => {
+ const streamingMessage = { ...assistantMessage, messageId: 'user-1_' } as TMessage;
+ renderBrowserAudio({ messages: [streamingMessage] });
+ await settle();
+
+ expect(spoken).toEqual([]);
+ });
+
+ it('does not speak the user message back to them', async () => {
+ const userMessage = { ...assistantMessage, isCreatedByUser: true } as TMessage;
+ renderBrowserAudio({ messages: [userMessage] });
+ await settle();
+
+ expect(spoken).toEqual([]);
+ });
+
+ it('cancels the utterance when the conversation is left', async () => {
+ const { unmount } = renderBrowserAudio();
+
+ await waitFor(() => expect(spoken).toEqual([responseText]));
+ unmount();
+
+ expect(cancelCount).toBeGreaterThan(0);
+ });
+});
diff --git a/client/src/hooks/Audio/MediaSourceAppender.ts b/client/src/hooks/Audio/MediaSourceAppender.ts
index 38c88ff188..cdcbf5467b 100644
--- a/client/src/hooks/Audio/MediaSourceAppender.ts
+++ b/client/src/hooks/Audio/MediaSourceAppender.ts
@@ -1,23 +1,56 @@
export class MediaSourceAppender {
private readonly mediaSource = new MediaSource();
private readonly audioChunks: ArrayBuffer[] = [];
+ private readonly type: string;
+ private objectUrl?: string;
private sourceBuffer?: SourceBuffer;
+ private isClosed = false;
constructor(type: string) {
- this.mediaSource.addEventListener('sourceopen', async () => {
- this.sourceBuffer = this.mediaSource.addSourceBuffer(type);
+ this.type = type;
+ this.mediaSource.addEventListener('sourceopen', () => {
+ if (this.sourceBuffer != null) {
+ return;
+ }
+ this.sourceBuffer = this.mediaSource.addSourceBuffer(this.type);
this.sourceBuffer.addEventListener('updateend', () => {
this.tryAppendNextChunk();
});
+
+ /** `sourceopen` only fires once a media element attaches the object URL, which
+ * can land after the whole response was already read. Draining here is what
+ * keeps short (fast) responses from stranding their chunks in the queue. */
+ this.tryAppendNextChunk();
});
}
private tryAppendNextChunk() {
- if (this.sourceBuffer != null && !this.sourceBuffer.updating && this.audioChunks.length > 0) {
- this.sourceBuffer.appendBuffer(this.audioChunks.shift()!);
+ if (this.sourceBuffer == null || this.sourceBuffer.updating) {
+ return;
+ }
+
+ const chunk = this.audioChunks.shift();
+ if (chunk == null) {
+ this.tryEndOfStream();
+ return;
+ }
+
+ this.sourceBuffer.appendBuffer(chunk);
+ }
+
+ private tryEndOfStream() {
+ /** `endOfStream()` only throws while the source is not open or a buffer is updating —
+ * an empty response still has to end, or the element waits on it forever. */
+ if (!this.isClosed || this.mediaSource.readyState !== 'open') {
+ return;
+ }
+ if (this.audioChunks.length > 0 || this.sourceBuffer?.updating === true) {
+ return;
}
+
+ this.mediaSource.endOfStream();
}
public addBase64Data(base64Data: string) {
@@ -31,13 +64,17 @@ export class MediaSourceAppender {
this.tryAppendNextChunk();
}
+ /**
+ * Signals that no further data will arrive. The media source is only ended once every
+ * queued chunk has been appended, so the tail of the audio is never truncated.
+ */
public close() {
- if (this.mediaSource.readyState === 'open') {
- this.mediaSource.endOfStream();
- }
+ this.isClosed = true;
+ this.tryEndOfStream();
}
public get mediaSourceUrl() {
- return URL.createObjectURL(this.mediaSource);
+ this.objectUrl ??= URL.createObjectURL(this.mediaSource);
+ return this.objectUrl;
}
}
diff --git a/client/src/hooks/Audio/__tests__/MediaSourceAppender.spec.ts b/client/src/hooks/Audio/__tests__/MediaSourceAppender.spec.ts
new file mode 100644
index 0000000000..635c0c2893
--- /dev/null
+++ b/client/src/hooks/Audio/__tests__/MediaSourceAppender.spec.ts
@@ -0,0 +1,164 @@
+import { MediaSourceAppender } from '../MediaSourceAppender';
+
+type Listener = () => void;
+
+class FakeSourceBuffer {
+ public updating = false;
+ public readonly appended: ArrayBuffer[] = [];
+ private readonly listeners = new Map<string, Listener[]>();
+
+ addEventListener(event: string, handler: Listener) {
+ const handlers = this.listeners.get(event) ?? [];
+ handlers.push(handler);
+ this.listeners.set(event, handlers);
+ }
+
+ appendBuffer(data: ArrayBuffer) {
+ this.updating = true;
+ this.appended.push(data);
+ queueMicrotask(() => {
+ this.updating = false;
+ this.listeners.get('updateend')?.forEach((handler) => handler());
+ });
+ }
+}
+
+const createdSources: FakeMediaSource[] = [];
+
+class FakeMediaSource {
+ public readyState: 'closed' | 'open' | 'ended' = 'closed';
+ public readonly sourceBuffers: FakeSourceBuffer[] = [];
+ private readonly listeners = new Map<string, Listener[]>();
+
+ constructor() {
+ createdSources.push(this);
+ }
+
+ addEventListener(event: string, handler: Listener) {
+ const handlers = this.listeners.get(event) ?? [];
+ handlers.push(handler);
+ this.listeners.set(event, handlers);
+ }
+
+ addSourceBuffer(_type: string) {
+ const sourceBuffer = new FakeSourceBuffer();
+ this.sourceBuffers.push(sourceBuffer);
+ return sourceBuffer;
+ }
+
+ endOfStream() {
+ this.readyState = 'ended';
+ }
+
+ /** A MediaSource stays `closed` until a media element attaches its object URL */
+ attach() {
+ this.readyState = 'open';
+ this.listeners.get('sourceopen')?.forEach((handler) => handler());
+ }
+}
+
+const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
+const chunk = (byte: number) => new Uint8Array([byte]).buffer as ArrayBuffer;
+const appendedBytes = (source: FakeMediaSource) =>
+ (source.sourceBuffers[0]?.appended ?? []).map((buffer) => new Uint8Array(buffer)[0]);
+
+describe('MediaSourceAppender', () => {
+ let objectUrlCount = 0;
+
+ beforeEach(() => {
+ createdSources.length = 0;
+ objectUrlCount = 0;
+ Object.defineProperty(global, 'MediaSource', {
+ writable: true,
+ configurable: true,
+ value: FakeMediaSource,
+ });
+ Object.defineProperty(global.URL, 'createObjectURL', {
+ writable: true,
+ configurable: true,
+ value: () => `blob:media-source-${++objectUrlCount}`,
+ });
+ });
+
+ it('appends chunks that were queued before the media element attached', async () => {
+ const appender = new MediaSourceAppender('audio/mpeg');
+ const source = createdSources[0];
+ appender.mediaSourceUrl;
+
+ /** A short response can be read in full before `sourceopen` fires */
+ appender.addData(chunk(1));
+ appender.addData(chunk(2));
+ expect(appendedBytes(source)).toEqual([]);
+
+ source.attach();
+ await flush();
+
+ expect(appendedBytes(source)).toEqual([1, 2]);
+ });
+
+ it('appends chunks that arrive after the media element attached', async () => {
+ const appender = new MediaSourceAppender('audio/mpeg');
+ const source = createdSources[0];
+ appender.mediaSourceUrl;
+
+ source.attach();
+ appender.addData(chunk(1));
+ await flush();
+ appender.addData(chunk(2));
+ await flush();
+
+ expect(appendedBytes(source)).toEqual([1, 2]);
+ });
+
+ it('only ends the stream once every queued chunk has been appended', async () => {
+ const appender = new MediaSourceAppender('audio/mpeg');
+ const source = createdSources[0];
+ appender.mediaSourceUrl;
+
+ appender.addData(chunk(1));
+ appender.addData(chunk(2));
+ appender.close();
+ expect(source.readyState).toBe('closed');
+
+ source.attach();
+ await flush();
+
+ expect(appendedBytes(source)).toEqual([1, 2]);
+ expect(source.readyState).toBe('ended');
+ });
+
+ it('ends a closed stream that never received data', async () => {
+ const appender = new MediaSourceAppender('audio/mpeg');
+ const source = createdSources[0];
+ appender.mediaSourceUrl;
+
+ /** An empty response, or a read timeout before the first byte, still has to end —
+ * otherwise the element waits on a source that can never receive data. */
+ source.attach();
+ appender.close();
+ await flush();
+
+ expect(source.readyState).toBe('ended');
+ });
+
+ it('ends a stream closed before the media element attached', async () => {
+ const appender = new MediaSourceAppender('audio/mpeg');
+ const source = createdSources[0];
+ appender.mediaSourceUrl;
+
+ appender.close();
+ expect(source.readyState).toBe('closed');
+
+ source.attach();
+ await flush();
+
+ expect(source.readyState).toBe('ended');
+ });
+
+ it('reuses a single object URL for the lifetime of the appender', () => {
+ const appender = new MediaSourceAppender('audio/mpeg');
+
+ expect(appender.mediaSourceUrl).toBe(appender.mediaSourceUrl);
+ expect(objectUrlCount).toBe(1);
+ });
+});
diff --git a/client/src/hooks/Audio/index.ts b/client/src/hooks/Audio/index.ts
index 15dfe6f746..fce85d220f 100644
--- a/client/src/hooks/Audio/index.ts
+++ b/client/src/hooks/Audio/index.ts
@@ -1,5 +1,6 @@
export * from './MediaSourceAppender';
export { default as useCustomAudioRef } from './useCustomAudioRef';
export { default as usePauseGlobalAudio } from './usePauseGlobalAudio';
+export { default as useAutoplayTrigger } from './useAutoplayTrigger';
export { default as useTTSExternal } from './useTTSExternal';
export { default as useTTSBrowser } from './useTTSBrowser';
diff --git a/client/src/hooks/Audio/useAutoplayTrigger.ts b/client/src/hooks/Audio/useAutoplayTrigger.ts
new file mode 100644
index 0000000000..ec257808f8
--- /dev/null
+++ b/client/src/hooks/Audio/useAutoplayTrigger.ts
@@ -0,0 +1,37 @@
+import { useRecoilValue } from 'recoil';
+import type { TMessage } from 'librechat-data-provider';
+import { useLatestMessage } from '~/hooks/Messages/useLatestMessage';
+import { getLatestText } from '~/utils';
+import store from '~/store';
+
+export type TAutoplayTrigger = {
+ /** Whether the latest assistant message is finalized and its run has not been played yet */
+ shouldPlay: boolean;
+ activeRunId: string | null;
+ latestMessage: TMessage | null;
+};
+
+/**
+ * Shared "Autoplay Latest Message" gate so every TTS engine autoplays on identical terms:
+ * the run must be finished, the branch tail must be a persisted assistant message carrying
+ * text, and its run must not have been played already.
+ */
+export default function useAutoplayTrigger(index: string | number = 0): TAutoplayTrigger {
+ const activeRunId = useRecoilValue(store.activeRunFamily(index));
+ const audioRunId = useRecoilValue(store.audioRunFamily(index));
+ const isSubmitting = useRecoilValue(store.isSubmittingFamily(index));
+ const latestMessage = useLatestMessage(index);
+
+ const shouldPlay = !!(
+ !isSubmitting &&
+ latestMessage &&
+ latestMessage.isCreatedByUser !== true &&
+ getLatestText(latestMessage) &&
+ latestMessage.messageId &&
+ !latestMessage.messageId.includes('_') &&
+ activeRunId != null &&
+ activeRunId !== audioRunId
+ );
+
+ return { shouldPlay, activeRunId, latestMessage };
+}
diff --git a/client/src/hooks/Input/useTextToSpeechBrowser.ts b/client/src/hooks/Input/useTextToSpeechBrowser.ts
index b7361c5ff5..a88f126aeb 100644
--- a/client/src/hooks/Input/useTextToSpeechBrowser.ts
+++ b/client/src/hooks/Input/useTextToSpeechBrowser.ts
@@ -1,4 +1,4 @@
-import { useMemo, useSyncExternalStore } from 'react';
+import { useMemo, useCallback, useSyncExternalStore } from 'react';
import { useRecoilValue } from 'recoil';
import type { VoiceOption } from '~/common';
import { subscribeSpeechVoices, getSpeechVoicesSnapshot } from '~/utils';
@@ -24,45 +24,52 @@ function useTextToSpeechBrowser({
return filteredVoices.map((v): VoiceOption => ({ value: v.name, label: v.name }));
}, [availableVoices, cloudBrowserVoices]);
- const generateSpeechLocal = (text: string) => {
- if (!isSpeechSynthesisSupported) {
- console.warn('Speech synthesis is not supported');
- return;
- }
+ /** Reports whether an utterance was actually queued: autoplay must not mark a run as
+ * played when the voice list has not loaded yet, or the message is never spoken. */
+ const generateSpeechLocal = useCallback(
+ (text: string): boolean => {
+ if (!isSpeechSynthesisSupported) {
+ console.warn('Speech synthesis is not supported');
+ return false;
+ }
- const synth = window.speechSynthesis;
- const voice = voices.find((v) => v.value === voiceName);
+ const synth = window.speechSynthesis;
+ const voice = voices.find((v) => v.value === voiceName);
- if (!voice) {
- console.warn('Selected voice not found');
- return;
- }
+ if (!voice) {
+ console.warn('Selected voice not found');
+ return false;
+ }
- try {
- synth.cancel();
- const utterance = new SpeechSynthesisUtterance(text);
- utterance.voice = synth.getVoices().find((v) => v.name === voice.value) || null;
- utterance.onend = () => {
- setIsSpeaking(false);
- };
- utterance.onerror = (event) => {
- if (event.error === 'interrupted' || event.error === 'canceled') {
+ try {
+ synth.cancel();
+ const utterance = new SpeechSynthesisUtterance(text);
+ utterance.voice = synth.getVoices().find((v) => v.name === voice.value) || null;
+ utterance.onend = () => {
setIsSpeaking(false);
- return;
- }
+ };
+ utterance.onerror = (event) => {
+ if (event.error === 'interrupted' || event.error === 'canceled') {
+ setIsSpeaking(false);
+ return;
+ }
- console.error('Speech synthesis error:', event);
+ console.error('Speech synthesis error:', event);
+ setIsSpeaking(false);
+ };
+ setIsSpeaking(true);
+ synth.speak(utterance);
+ return true;
+ } catch (error) {
+ console.error('Error generating speech:', error);
setIsSpeaking(false);
- };
- setIsSpeaking(true);
- synth.speak(utterance);
- } catch (error) {
- console.error('Error generating speech:', error);
- setIsSpeaking(false);
- }
- };
+ return false;
+ }
+ },
+ [isSpeechSynthesisSupported, voices, voiceName, setIsSpeaking],
+ );
- const cancelSpeechLocal = () => {
+ const cancelSpeechLocal = useCallback(() => {
if (!isSpeechSynthesisSupported) {
return;
}
@@ -74,7 +81,7 @@ function useTextToSpeechBrowser({
} finally {
setIsSpeaking(false);
}
- };
+ }, [isSpeechSynthesisSupported, setIsSpeaking]);
return { generateSpeechLocal, cancelSpeechLocal, voices, isSpeechSynthesisSupported };
}