Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔇 fix: Keep Reasoning Out of Read Aloud (#16790)

Marco Beretta9068f0e92215
diff --git a/api/server/services/Files/Audio/streamAudio.js b/api/server/services/Files/Audio/streamAudio.js
index 7120399b5e..ab7c7d9c3c 100644
--- a/api/server/services/Files/Audio/streamAudio.js
+++ b/api/server/services/Files/Audio/streamAudio.js
@@ -1,11 +1,5 @@
-const { scopedCacheKey } = require('@librechat/data-schemas');
-const {
- Time,
- CacheKeys,
- SEPARATORS,
- parseTextParts,
- findLastSeparatorIndex,
-} = require('librechat-data-provider');
+const { createSpeechChunkProcessor } = require('@librechat/api');
+const { CacheKeys, SEPARATORS, findLastSeparatorIndex } = require('librechat-data-provider');
const { getLogStores } = require('~/cache');
const { getMessage } = require('~/models');
@@ -51,88 +45,18 @@ function getRandomVoiceId(voiceIds) {
* @property {string[]} normalizedAlignment.chars
*/
-const MAX_NOT_FOUND_COUNT = 6;
-const MAX_NO_CHANGE_COUNT = 10;
-
/**
* @param {string} user
* @param {string} messageId
- * @returns {() => Promise<{ text: string, isFinished: boolean }[]>}
+ * @returns {() => Promise<{ text: string, isFinished: boolean }[] | string>}
*/
function createChunkProcessor(user, messageId) {
- let notFoundCount = 0;
- let noChangeCount = 0;
- let processedText = '';
- if (!messageId) {
- throw new Error('Message ID is required');
- }
-
- const messageCache = getLogStores(CacheKeys.MESSAGES);
- // Captured at creation time — must be called within an active request ALS scope
- const cacheKey = scopedCacheKey(messageId);
-
- /**
- * @returns {Promise<{ text: string, isFinished: boolean }[] | string>}
- */
- async function processChunks() {
- if (notFoundCount >= MAX_NOT_FOUND_COUNT) {
- return `Message not found after ${MAX_NOT_FOUND_COUNT} attempts`;
- }
-
- if (noChangeCount >= MAX_NO_CHANGE_COUNT) {
- return `No change in message after ${MAX_NO_CHANGE_COUNT} attempts`;
- }
-
- /** @type { string | { text: string; complete: boolean } } */
- let message = await messageCache.get(cacheKey);
- if (!message) {
- message = await getMessage({ user, messageId });
- }
-
- if (!message) {
- notFoundCount++;
- return [];
- } else {
- const text = message.content?.length > 0 ? parseTextParts(message.content) : message.text;
- messageCache.set(
- cacheKey,
- {
- text,
- complete: true,
- },
- Time.FIVE_MINUTES,
- );
- }
-
- const text = typeof message === 'string' ? message : message.text;
- const complete = typeof message === 'string' ? false : (message.complete ?? true);
-
- if (text === processedText) {
- noChangeCount++;
- }
-
- const remainingText = text.slice(processedText.length);
- const chunks = [];
-
- if (!complete && remainingText.length >= 20) {
- const separatorIndex = findLastSeparatorIndex(remainingText);
- if (separatorIndex !== -1) {
- const chunkText = remainingText.slice(0, separatorIndex + 1);
- chunks.push({ text: chunkText, isFinished: false });
- processedText += chunkText;
- } else {
- chunks.push({ text: remainingText, isFinished: false });
- processedText = text;
- }
- } else if (complete && remainingText.trim().length > 0) {
- chunks.push({ text: remainingText.trim(), isFinished: true });
- processedText = text;
- }
-
- return chunks;
- }
-
- return processChunks;
+ return createSpeechChunkProcessor({
+ user,
+ messageId,
+ cache: getLogStores(CacheKeys.MESSAGES),
+ getMessage,
+ });
}
/**
diff --git a/api/server/services/Files/Audio/streamAudio.spec.js b/api/server/services/Files/Audio/streamAudio.spec.js
index 977d8730aa..021e8373de 100644
--- a/api/server/services/Files/Audio/streamAudio.spec.js
+++ b/api/server/services/Files/Audio/streamAudio.spec.js
@@ -1,4 +1,4 @@
-const { createChunkProcessor, splitTextIntoChunks } = require('./streamAudio');
+const { splitTextIntoChunks } = require('./streamAudio');
jest.mock('keyv');
@@ -55,102 +55,6 @@ jest.mock('~/cache/getLogStores', () => {
});
});
-describe('processChunks', () => {
- let processChunks;
- let mockMessageCache;
-
- beforeEach(() => {
- jest.resetAllMocks();
- mockMessageCache = {
- get: jest.fn(),
- set: jest.fn(),
- };
- require('~/cache/getLogStores').mockReturnValue(mockMessageCache);
- processChunks = createChunkProcessor('userId', 'message-id');
- });
-
- it('should return an empty array when the message is not found', async () => {
- mockMessageCache.get.mockResolvedValueOnce(null);
-
- const result = await processChunks();
-
- expect(result).toEqual([]);
- expect(mockMessageCache.get).toHaveBeenCalledWith('message-id');
- });
-
- it('should return an error message after MAX_NOT_FOUND_COUNT attempts', async () => {
- mockMessageCache.get.mockResolvedValue(null);
-
- for (let i = 0; i < 6; i++) {
- await processChunks();
- }
- const result = await processChunks();
-
- expect(result).toBe('Message not found after 6 attempts');
- });
-
- it('should return chunks for an incomplete message with separators', async () => {
- const messageText = 'This is a long message. It should be split into chunks. Lol hi mom';
- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: false });
-
- const result = await processChunks();
-
- expect(result).toEqual([
- { text: 'This is a long message. It should be split into chunks.', isFinished: false },
- ]);
- });
-
- it('should return chunks for an incomplete message without separators', async () => {
- const messageText = 'This is a long message without separators hello there my friend';
- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: false });
-
- const result = await processChunks();
-
- expect(result).toEqual([{ text: messageText, isFinished: false }]);
- });
-
- it('should return the remaining text as a chunk for a complete message', async () => {
- const messageText = 'This is a finished message.';
- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: true });
-
- const result = await processChunks();
-
- expect(result).toEqual([{ text: messageText, isFinished: true }]);
- });
-
- it('should return an empty array for a complete message with no remaining text', async () => {
- const messageText = 'This is a finished message.';
- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: true });
-
- await processChunks();
- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: true });
- const result = await processChunks();
-
- expect(result).toEqual([]);
- });
-
- it('should return an error message after MAX_NO_CHANGE_COUNT attempts with no change', async () => {
- const messageText = 'This is a message that does not change.';
- mockMessageCache.get.mockResolvedValue({ text: messageText, complete: false });
-
- for (let i = 0; i < 11; i++) {
- await processChunks();
- }
- const result = await processChunks();
-
- expect(result).toBe('No change in message after 10 attempts');
- });
-
- it('should handle string messages as incomplete', async () => {
- const messageText = 'This is a message as a string.';
- mockMessageCache.get.mockResolvedValueOnce(messageText);
-
- const result = await processChunks();
-
- expect(result).toEqual([{ text: messageText, isFinished: false }]);
- });
-});
-
describe('splitTextIntoChunks', () => {
test('splits text into chunks of specified size with default separators', () => {
const text = 'This is a test. This is only a test! Make sure it works properly? Okay.';
diff --git a/client/src/components/Chat/Input/BrowserAudio.tsx b/client/src/components/Chat/Input/BrowserAudio.tsx
index 262310f7b2..1da7af5073 100644
--- a/client/src/components/Chat/Input/BrowserAudio.tsx
+++ b/client/src/components/Chat/Input/BrowserAudio.tsx
@@ -1,7 +1,7 @@
import { useEffect } from 'react';
import { useSetRecoilState } from 'recoil';
import { useParams } from 'react-router-dom';
-import { parseTextParts } from 'librechat-data-provider';
+import { getSpeechText } from 'librechat-data-provider';
import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';
import useAutoplayTrigger from '~/hooks/Audio/useAutoplayTrigger';
import { logger } from '~/utils';
@@ -36,10 +36,7 @@ export default function BrowserAudio({ index = 0 }) {
return;
}
- const text =
- Array.isArray(latestMessage.content) && latestMessage.content.length > 0
- ? parseTextParts(latestMessage.content)
- : (latestMessage.text ?? '');
+ const text = getSpeechText(latestMessage);
if (!text) {
return;
diff --git a/client/src/components/Chat/Input/StreamAudio.tsx b/client/src/components/Chat/Input/StreamAudio.tsx
index d7e788da46..5db4b9c767 100644
--- a/client/src/components/Chat/Input/StreamAudio.tsx
+++ b/client/src/components/Chat/Input/StreamAudio.tsx
@@ -1,7 +1,7 @@
import { useEffect, useCallback } from 'react';
import { useParams } from 'react-router-dom';
-import { QueryKeys } from 'librechat-data-provider';
import { useQueryClient } from '@tanstack/react-query';
+import { QueryKeys, getSpeechText } from 'librechat-data-provider';
import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';
import type { TMessage } from 'librechat-data-provider';
import {
@@ -66,7 +66,9 @@ export default function StreamAudio({ index = 0 }) {
setGlobalAudioURL(null);
}
- let cacheKey = latestMessage?.text ?? '';
+ /** Keyed by the spoken text, as manual playback is, so audio cached from an
+ * unfiltered message (reasoning included) is never replayed. */
+ let cacheKey = latestMessage ? getSpeechText(latestMessage) : '';
const cache = await caches.open('tts-responses');
const cachedResponse = await cache.match(cacheKey);
@@ -139,7 +141,7 @@ export default function StreamAudio({ index = 0 }) {
const targetMessage = latestMessages.find(
(msg) => msg.messageId === latestMessage?.messageId,
);
- cacheKey = targetMessage?.text ?? '';
+ cacheKey = targetMessage ? getSpeechText(targetMessage) : '';
if (!cacheKey) {
logger.warn('Cache key not found, skipping audio cache');
} else {
diff --git a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx
index 3dcb631a07..b14a53ae6a 100644
--- a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx
+++ b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx
@@ -1,9 +1,9 @@
import React from 'react';
import { RecoilRoot } from 'recoil';
import { render, waitFor } from '@testing-library/react';
-import { Constants, QueryKeys } from 'librechat-data-provider';
import { MemoryRouter, Routes, Route } from 'react-router-dom';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider';
import type { TMessage, TConversation } from 'librechat-data-provider';
import BrowserAudio from '../BrowserAudio';
import store from '~/store';
@@ -108,6 +108,23 @@ describe('BrowserAudio autoplay', () => {
await waitFor(() => expect(spoken).toEqual([responseText]));
});
+ it('speaks the answer without the reasoning that preceded it', async () => {
+ renderBrowserAudio({
+ messages: [
+ {
+ ...assistantMessage,
+ text: '',
+ content: [
+ { type: ContentTypes.THINK, think: 'Recalling European capitals.' },
+ { type: ContentTypes.TEXT, text: responseText },
+ ],
+ },
+ ],
+ });
+
+ await waitFor(() => expect(spoken).toEqual([responseText]));
+ });
+
it('does not speak while the run is still submitting', async () => {
renderBrowserAudio({ isSubmitting: true });
await settle();
@@ -130,6 +147,22 @@ describe('BrowserAudio autoplay', () => {
expect(spoken).toEqual([]);
});
+ /** A turn stopped mid-reasoning persists its reasoning as `text` beside the parts. */
+ it('does not autoplay a stopped turn that only reasoned', async () => {
+ renderBrowserAudio({
+ messages: [
+ {
+ ...assistantMessage,
+ text: 'Recalling European capitals.',
+ content: [{ type: ContentTypes.THINK, think: 'Recalling European capitals.' }],
+ },
+ ],
+ });
+ await settle();
+
+ expect(spoken).toEqual([]);
+ });
+
it('does not speak the user message back to them', async () => {
const userMessage = { ...assistantMessage, isCreatedByUser: true } as TMessage;
renderBrowserAudio({ messages: [userMessage] });
diff --git a/client/src/components/Chat/Input/__tests__/StreamAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/StreamAudio.spec.tsx
new file mode 100644
index 0000000000..3ddf54f75a
--- /dev/null
+++ b/client/src/components/Chat/Input/__tests__/StreamAudio.spec.tsx
@@ -0,0 +1,64 @@
+import React from 'react';
+import { RecoilRoot } from 'recoil';
+import { render, waitFor } from '@testing-library/react';
+import { MemoryRouter, Routes, Route } from 'react-router-dom';
+import { Constants, QueryKeys } from 'librechat-data-provider';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import type { TMessage, TConversation } from 'librechat-data-provider';
+import StreamAudio from '../StreamAudio';
+import store from '~/store';
+
+jest.mock('~/hooks', () => ({ useAuthContext: () => ({ token: 'token' }) }));
+
+const conversationId = 'convo-1';
+
+const match = jest.fn(async (_key: string) => ({ blob: async () => new Blob() }));
+
+describe('StreamAudio cache', () => {
+ beforeAll(() => {
+ Object.defineProperty(global, 'caches', {
+ writable: true,
+ configurable: true,
+ value: { open: async () => ({ match, put: jest.fn() }) },
+ });
+ URL.createObjectURL = jest.fn(() => 'blob:audio');
+ URL.revokeObjectURL = jest.fn();
+ });
+
+ /** Audio cached before reasoning was filtered is keyed by the full text; looking up the
+ * spoken answer instead means that recording is never replayed. */
+ it('looks up cached audio by the spoken answer, not the stored text', async () => {
+ const message = {
+ messageId: 'assistant-1',
+ conversationId,
+ parentMessageId: Constants.NO_PARENT,
+ isCreatedByUser: false,
+ text: ':::thinking\nRecalling European capitals.\n:::\nThe capital of Germany is Berlin.',
+ } as TMessage;
+ const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ queryClient.setQueryData([QueryKeys.messages, conversationId], [message]);
+
+ render(
+ <QueryClientProvider client={queryClient}>
+ <MemoryRouter initialEntries={[`/c/${conversationId}`]}>
+ <RecoilRoot
+ initializeState={({ set }) => {
+ set(store.automaticPlayback, true);
+ set(store.conversationByIndex(0), { conversationId } as TConversation);
+ set(store.activeRunFamily(0), 'run-1');
+ set(store.audioRunFamily(0), null);
+ set(store.isSubmittingFamily(0), false);
+ }}
+ >
+ <Routes>
+ <Route path="/c/:conversationId" element={<StreamAudio index={0} />} />
+ </Routes>
+ </RecoilRoot>
+ </MemoryRouter>
+ </QueryClientProvider>,
+ );
+
+ await waitFor(() => expect(match).toHaveBeenCalled());
+ expect(match).toHaveBeenCalledWith('The capital of Germany is Berlin.');
+ });
+});
diff --git a/client/src/components/Chat/Messages/Content/MessageContent.tsx b/client/src/components/Chat/Messages/Content/MessageContent.tsx
index 6cea94d3ac..4502cef6ad 100644
--- a/client/src/components/Chat/Messages/Content/MessageContent.tsx
+++ b/client/src/components/Chat/Messages/Content/MessageContent.tsx
@@ -1,7 +1,7 @@
import { memo, Suspense, useMemo } from 'react';
import { useRecoilValue } from 'recoil';
-import { Constants } from 'librechat-data-provider';
import { Alert, DelayedRender } from '@librechat/client';
+import { Constants, parseThinkingContent } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
import type { TMessageContentProps, TDisplayProps } from '~/common';
import useSmoothStreaming from '~/hooks/Messages/useSmoothStreaming';
@@ -23,14 +23,6 @@ const ERROR_CONNECTION_TEXT = 'Error connecting to server, try refreshing the pa
const DELAYED_ERROR_TIMEOUT = 5500;
const UNFINISHED_DELAY = 250;
-const parseThinkingContent = (text: string) => {
- const thinkingMatch = text.match(/:::thinking([\s\S]*?):::/);
- return {
- thinkingContent: thinkingMatch ? thinkingMatch[1].trim() : '',
- regularContent: thinkingMatch ? text.replace(/:::thinking[\s\S]*?:::/, '').trim() : text,
- };
-};
-
const LoadingFallback = () => (
<div className="mb-[0.625rem]">
<EmptyText underHeaderIcon />
diff --git a/client/src/components/Chat/Messages/HoverButtons.tsx b/client/src/components/Chat/Messages/HoverButtons.tsx
index d608ea7747..225daa9bf3 100644
--- a/client/src/components/Chat/Messages/HoverButtons.tsx
+++ b/client/src/components/Chat/Messages/HoverButtons.tsx
@@ -2,7 +2,7 @@ import React, { useState, useMemo, useCallback, memo } from 'react';
import { Copy, Check } from 'lucide';
import { useAtomValue } from 'jotai';
import { useRecoilState } from 'recoil';
-import { findMessageById, isUserInitiatedCompaction } from 'librechat-data-provider';
+import { getSpeechText, findMessageById, isUserInitiatedCompaction } from 'librechat-data-provider';
import {
Button,
EditIcon,
@@ -55,38 +55,6 @@ type HoverButtonProps = {
disabled?: boolean;
};
-const extractMessageContent = (message: TMessage): string => {
- if (typeof message.content === 'string') {
- return message.content;
- }
-
- if (Array.isArray(message.content)) {
- return message.content
- .map((part) => {
- if (part == null) {
- return '';
- }
- if (typeof part === 'string') {
- return part;
- }
- if ('text' in part) {
- return part.text || '';
- }
- if ('think' in part) {
- const think = part.think;
- if (typeof think === 'string') {
- return think;
- }
- return think && 'text' in think ? think.text || '' : '';
- }
- return '';
- })
- .join('');
- }
-
- return message.text || '';
-};
-
const HoverButton = memo(
({
id,
@@ -226,16 +194,17 @@ const HoverButtons = ({
};
const handleCopy = () => copyToClipboard(setIsCopied);
+ const speechText = getSpeechText(message);
return (
<div className="group visible flex justify-center gap-0.5 self-end focus-within:outline-hidden lg:justify-start">
{/* Text to Speech */}
- {TextToSpeech && !error && !isActiveStreamingMessage && (
+ {TextToSpeech && !error && !isActiveStreamingMessage && speechText.length > 0 && (
<MessageAudio
index={index}
isLast={isLast}
messageId={message.messageId}
- content={extractMessageContent(message)}
+ content={speechText}
renderButton={(props) => (
<HoverButton
onClick={props.onClick}
diff --git a/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx b/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx
index 55cd394c0f..4dfdc6a49a 100644
--- a/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx
+++ b/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx
@@ -1,7 +1,7 @@
import React from 'react';
import { MemoryRouter } from 'react-router-dom';
import { RecoilRoot, type MutableSnapshot } from 'recoil';
-import { act, render, screen } from '@testing-library/react';
+import { act, fireEvent, render, screen } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import {
Constants,
@@ -42,6 +42,7 @@ function renderHoverButtons({
getCanCopy = () => hasCopyableText({ text: message.text, content: message.content }),
handleFeedback,
thread,
+ voice,
}: {
isSubmitting: boolean;
message?: TMessage;
@@ -53,12 +54,20 @@ function renderHoverButtons({
/** The rows the hover controls resolve the message's parent from. Omitted, the
* thread is unavailable, as on a search row. */
thread?: TMessage[];
+ /** Enables Read Aloud on the browser engine with this voice selected. */
+ voice?: string;
}) {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
});
- const initializeState = ({ set }: MutableSnapshot) => set(store.textToSpeech, false);
+ const initializeState = ({ set }: MutableSnapshot) => {
+ set(store.textToSpeech, voice != null);
+ if (voice != null) {
+ set(store.voice, voice);
+ set(store.speechSettingsInitialized, true);
+ }
+ };
const { container } = render(
<QueryClientProvider client={queryClient}>
@@ -639,3 +648,87 @@ describe('HoverButtons feedback affordance', () => {
expect(screen.getByTestId('copy-response-button')).toBeInTheDocument();
});
});
+
+describe('HoverButtons read aloud', () => {
+ const voiceName = 'Test Voice';
+ const spoken: string[] = [];
+
+ class FakeSpeechSynthesisUtterance {
+ public voice: SpeechSynthesisVoice | null = null;
+ public onend: (() => void) | null = null;
+ public onerror: (() => void) | null = null;
+ constructor(public text: string) {}
+ }
+
+ beforeAll(() => {
+ /** jsdom lacks it; the message's audio element revokes its source on unmount. */
+ URL.revokeObjectURL = jest.fn();
+ Object.defineProperty(window, 'speechSynthesis', {
+ writable: true,
+ configurable: true,
+ value: {
+ getVoices: () => [{ name: voiceName, localService: true }],
+ addEventListener: () => undefined,
+ speak: (utterance: FakeSpeechSynthesisUtterance) => spoken.push(utterance.text),
+ cancel: () => undefined,
+ },
+ });
+ Object.defineProperty(global, 'SpeechSynthesisUtterance', {
+ writable: true,
+ configurable: true,
+ value: FakeSpeechSynthesisUtterance,
+ });
+ });
+
+ beforeEach(() => {
+ spoken.length = 0;
+ });
+
+ const readAloud = (message: Partial<TMessage>) => {
+ renderHoverButtons({
+ isSubmitting: false,
+ message: { ...userMessage, messageId: 'assistant-1', isCreatedByUser: false, ...message },
+ isLast: true,
+ latestMessageId: 'assistant-1',
+ voice: voiceName,
+ });
+ fireEvent.click(screen.getByTestId('read-aloud-button'));
+ };
+
+ it('speaks the answer without the reasoning parts before it', () => {
+ readAloud({
+ text: '',
+ content: [
+ { type: ContentTypes.THINK, think: 'Let me work out 2 + 2 first.' },
+ { type: ContentTypes.TEXT, text: 'The answer is 4.' },
+ ],
+ });
+
+ expect(spoken).toEqual(['The answer is 4.']);
+ });
+
+ it('offers no read aloud for a response that only reasoned', () => {
+ renderHoverButtons({
+ isSubmitting: false,
+ message: {
+ ...userMessage,
+ messageId: 'assistant-1',
+ isCreatedByUser: false,
+ text: 'Let me work out 2 + 2 first.',
+ content: [{ type: ContentTypes.THINK, think: 'Let me work out 2 + 2 first.' }],
+ },
+ isLast: true,
+ latestMessageId: 'assistant-1',
+ voice: voiceName,
+ });
+
+ expect(screen.queryByTestId('read-aloud-button')).toBeNull();
+ expect(screen.getByTestId('copy-response-button')).toBeInTheDocument();
+ });
+
+ it('speaks the answer without a legacy thinking block in plain text', () => {
+ readAloud({ text: ':::thinking\nLet me work out 2 + 2 first.\n:::\nThe answer is 4.' });
+
+ expect(spoken).toEqual(['The answer is 4.']);
+ });
+});
diff --git a/client/src/hooks/Audio/__tests__/useAutoplayTrigger.spec.tsx b/client/src/hooks/Audio/__tests__/useAutoplayTrigger.spec.tsx
new file mode 100644
index 0000000000..2528440e0d
--- /dev/null
+++ b/client/src/hooks/Audio/__tests__/useAutoplayTrigger.spec.tsx
@@ -0,0 +1,63 @@
+import React from 'react';
+import { RecoilRoot } from 'recoil';
+import { renderHook } from '@testing-library/react';
+import { MemoryRouter, Routes, Route } from 'react-router-dom';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider';
+import type { TMessage, TConversation } from 'librechat-data-provider';
+import useAutoplayTrigger from '../useAutoplayTrigger';
+import store from '~/store';
+
+const conversationId = 'convo-1';
+
+const assistantMessage = {
+ messageId: 'assistant-1',
+ conversationId,
+ parentMessageId: Constants.NO_PARENT,
+ isCreatedByUser: false,
+ text: 'The capital of Germany is Berlin.',
+} as TMessage;
+
+const renderTrigger = (message: TMessage) => {
+ const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+ queryClient.setQueryData([QueryKeys.messages, conversationId], [message]);
+
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ <QueryClientProvider client={queryClient}>
+ <MemoryRouter initialEntries={[`/c/${conversationId}`]}>
+ <RecoilRoot
+ initializeState={({ set }) => {
+ set(store.conversationByIndex(0), { conversationId } as TConversation);
+ set(store.activeRunFamily(0), 'run-1');
+ set(store.audioRunFamily(0), null);
+ set(store.isSubmittingFamily(0), false);
+ }}
+ >
+ <Routes>
+ <Route path="/c/:conversationId" element={<>{children}</>} />
+ </Routes>
+ </RecoilRoot>
+ </MemoryRouter>
+ </QueryClientProvider>
+ );
+
+ return renderHook(() => useAutoplayTrigger(0), { wrapper }).result;
+};
+
+describe('useAutoplayTrigger', () => {
+ it('plays a finalized answer', () => {
+ expect(renderTrigger(assistantMessage).current.shouldPlay).toBe(true);
+ });
+
+ /** A turn stopped mid-reasoning persists its reasoning as `text` beside the parts; playing
+ * it would open a server stream with nothing to speak. */
+ it('does not play a stopped turn that only reasoned', () => {
+ const result = renderTrigger({
+ ...assistantMessage,
+ text: 'Recalling European capitals.',
+ content: [{ type: ContentTypes.THINK, think: 'Recalling European capitals.' }],
+ });
+
+ expect(result.current.shouldPlay).toBe(false);
+ });
+});
diff --git a/client/src/hooks/Audio/useAutoplayTrigger.ts b/client/src/hooks/Audio/useAutoplayTrigger.ts
index ec257808f8..67049d58cc 100644
--- a/client/src/hooks/Audio/useAutoplayTrigger.ts
+++ b/client/src/hooks/Audio/useAutoplayTrigger.ts
@@ -1,7 +1,7 @@
import { useRecoilValue } from 'recoil';
+import { getSpeechText } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
import { useLatestMessage } from '~/hooks/Messages/useLatestMessage';
-import { getLatestText } from '~/utils';
import store from '~/store';
export type TAutoplayTrigger = {
@@ -14,7 +14,7 @@ export type TAutoplayTrigger = {
/**
* Shared "Autoplay Latest Message" gate so every TTS engine autoplays on identical terms:
* the run must be finished, the branch tail must be a persisted assistant message carrying
- * text, and its run must not have been played already.
+ * speakable text (reasoning alone is not), and its run must not have been played already.
*/
export default function useAutoplayTrigger(index: string | number = 0): TAutoplayTrigger {
const activeRunId = useRecoilValue(store.activeRunFamily(index));
@@ -26,7 +26,7 @@ export default function useAutoplayTrigger(index: string | number = 0): TAutopla
!isSubmitting &&
latestMessage &&
latestMessage.isCreatedByUser !== true &&
- getLatestText(latestMessage) &&
+ getSpeechText(latestMessage) &&
latestMessage.messageId &&
!latestMessage.messageId.includes('_') &&
activeRunId != null &&
diff --git a/client/src/hooks/Audio/useTTSBrowser.ts b/client/src/hooks/Audio/useTTSBrowser.ts
index 9edd9a06ec..215705565c 100644
--- a/client/src/hooks/Audio/useTTSBrowser.ts
+++ b/client/src/hooks/Audio/useTTSBrowser.ts
@@ -61,7 +61,9 @@ const useTTSBrowser = (props?: TUseTextToSpeech) => {
if (isMouseDownRef.current) {
const messageContent = content ?? '';
const parsedMessage =
- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ typeof messageContent === 'string'
+ ? messageContent
+ : parseTextParts(messageContent, true);
generateSpeech(parsedMessage);
}
}, 1000);
@@ -81,7 +83,7 @@ const useTTSBrowser = (props?: TUseTextToSpeech) => {
} else {
const messageContent = content ?? '';
const parsedMessage =
- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent, true);
generateSpeech(parsedMessage);
}
};
diff --git a/client/src/hooks/Audio/useTTSExternal.ts b/client/src/hooks/Audio/useTTSExternal.ts
index d296abf4a4..634e877489 100644
--- a/client/src/hooks/Audio/useTTSExternal.ts
+++ b/client/src/hooks/Audio/useTTSExternal.ts
@@ -62,7 +62,9 @@ const useTTSExternal = (props?: TUseTextToSpeech) => {
if (isMouseDownRef.current) {
const messageContent = content ?? '';
const parsedMessage =
- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ typeof messageContent === 'string'
+ ? messageContent
+ : parseTextParts(messageContent, true);
generateSpeech(parsedMessage, false);
}
}, 1000);
@@ -82,7 +84,7 @@ const useTTSExternal = (props?: TUseTextToSpeech) => {
} else {
const messageContent = content ?? '';
const parsedMessage =
- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent, true);
generateSpeech(parsedMessage, false);
}
};
diff --git a/client/src/hooks/Input/useTextToSpeech.ts b/client/src/hooks/Input/useTextToSpeech.ts
index 12da8ef65c..c5eff56a5b 100644
--- a/client/src/hooks/Input/useTextToSpeech.ts
+++ b/client/src/hooks/Input/useTextToSpeech.ts
@@ -1,5 +1,5 @@
-import { useRecoilState, useRecoilValue } from 'recoil';
import { useRef, useMemo, useEffect, useState } from 'react';
+import { useRecoilState, useRecoilValue } from 'recoil';
import { parseTextParts } from 'librechat-data-provider';
import type { TMessageContentParts } from 'librechat-data-provider';
import type { Option } from '~/common';
@@ -120,7 +120,9 @@ const useTextToSpeech = (props?: TUseTextToSpeech) => {
if (isMouseDownRef.current) {
const messageContent = content ?? '';
const parsedMessage =
- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ typeof messageContent === 'string'
+ ? messageContent
+ : parseTextParts(messageContent, true);
generateSpeech(parsedMessage, false);
}
}, 1000);
@@ -140,7 +142,7 @@ const useTextToSpeech = (props?: TUseTextToSpeech) => {
} else {
const messageContent = content ?? '';
const parsedMessage =
- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent, true);
generateSpeech(parsedMessage, false);
}
};
diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts
index fb20a690ed..66226ed4b4 100644
--- a/packages/api/src/files/index.ts
+++ b/packages/api/src/files/index.ts
@@ -22,6 +22,7 @@ export * from './rag/context';
export * from './rag/search';
export * from './regexEngine';
export * from './retention';
+export * from './speech';
export * from './sse';
export * from './sweep';
export * from './usage';
diff --git a/packages/api/src/files/speech.spec.ts b/packages/api/src/files/speech.spec.ts
new file mode 100644
index 0000000000..b939bac4da
--- /dev/null
+++ b/packages/api/src/files/speech.spec.ts
@@ -0,0 +1,116 @@
+import { ContentTypes } from 'librechat-data-provider';
+import type { TMessage } from 'librechat-data-provider';
+import type { SpeechMessageCache } from './speech';
+import { createSpeechChunkProcessor } from './speech';
+
+const messageId = 'message-id';
+
+function setup(stored: TMessage | null = null) {
+ const entries = new Map<string, Parameters<SpeechMessageCache['set']>[1]>();
+ const cache = {
+ get: jest.fn(async (key: string) => entries.get(key)),
+ set: jest.fn((key: string, value: Parameters<SpeechMessageCache['set']>[1]) =>
+ entries.set(key, value),
+ ),
+ };
+ const getMessage = jest.fn(async () => stored);
+ const processChunks = createSpeechChunkProcessor({
+ user: 'user-id',
+ messageId,
+ cache,
+ getMessage,
+ });
+ return { cache, getMessage, processChunks };
+}
+
+describe('createSpeechChunkProcessor', () => {
+ it('requires a message id', () => {
+ expect(() =>
+ createSpeechChunkProcessor({
+ user: 'user-id',
+ cache: { get: jest.fn(), set: jest.fn() },
+ getMessage: jest.fn(),
+ }),
+ ).toThrow('Message ID is required');
+ });
+
+ it('returns no chunks while the message is not found', async () => {
+ const { cache, getMessage, processChunks } = setup();
+
+ expect(await processChunks()).toEqual([]);
+ expect(cache.get).toHaveBeenCalledWith(messageId);
+ expect(getMessage).toHaveBeenCalledWith({ user: 'user-id', messageId });
+ });
+
+ it('gives up after the message stays missing', async () => {
+ const { processChunks } = setup();
+
+ for (let i = 0; i < 6; i++) {
+ await processChunks();
+ }
+
+ expect(await processChunks()).toBe('Message not found after 6 attempts');
+ });
+
+ it('splits an incomplete message at its last separator', async () => {
+ const { cache, processChunks } = setup();
+ cache.get.mockResolvedValueOnce({
+ text: 'This is a long message. It should be split into chunks. Lol hi mom',
+ complete: false,
+ });
+
+ expect(await processChunks()).toEqual([
+ { text: 'This is a long message. It should be split into chunks.', isFinished: false },
+ ]);
+ });
+
+ it('sends an incomplete message without separators whole', async () => {
+ const text = 'This is a long message without separators hello there my friend';
+ const { cache, processChunks } = setup();
+ cache.get.mockResolvedValueOnce({ text, complete: false });
+
+ expect(await processChunks()).toEqual([{ text, isFinished: false }]);
+ });
+
+ it('finishes with the remaining text of a complete message', async () => {
+ const text = 'This is a finished message.';
+ const { cache, processChunks } = setup();
+ cache.get.mockResolvedValueOnce({ text, complete: true });
+
+ expect(await processChunks()).toEqual([{ text, isFinished: true }]);
+ expect(await processChunks()).toEqual([]);
+ });
+
+ it('gives up after the message stops changing', async () => {
+ const { cache, processChunks } = setup();
+ cache.get.mockResolvedValue({ text: 'This message does not change.', complete: false });
+
+ for (let i = 0; i < 11; i++) {
+ await processChunks();
+ }
+
+ expect(await processChunks()).toBe('No change in message after 10 attempts');
+ });
+
+ it('treats a cached string as incomplete', async () => {
+ const text = 'This is a message as a string.';
+ const { cache, processChunks } = setup();
+ cache.get.mockResolvedValueOnce(text);
+
+ expect(await processChunks()).toEqual([{ text, isFinished: false }]);
+ });
+
+ /** An aborted turn persists `text` with its reasoning alongside the content parts. */
+ it('never speaks the reasoning of a stored message', async () => {
+ const { processChunks } = setup({
+ messageId,
+ text: 'Let me work this out. The answer is 4.',
+ content: [
+ { type: ContentTypes.THINK, think: 'Let me work this out.' },
+ { type: ContentTypes.TEXT, text: 'The answer is 4.' },
+ ],
+ } as TMessage);
+
+ expect(await processChunks()).toEqual([{ text: 'The answer is 4.', isFinished: true }]);
+ });
+});
diff --git a/packages/api/src/files/speech.ts b/packages/api/src/files/speech.ts
new file mode 100644
index 0000000000..4b414be1e9
--- /dev/null
+++ b/packages/api/src/files/speech.ts
@@ -0,0 +1,89 @@
+import { scopedCacheKey } from '@librechat/data-schemas';
+import { Time, getSpeechText, findLastSeparatorIndex } from 'librechat-data-provider';
+import type { TMessage } from 'librechat-data-provider';
+
+const MAX_NOT_FOUND_COUNT = 6;
+const MAX_NO_CHANGE_COUNT = 10;
+
+export type SpeechChunk = { text: string; isFinished: boolean };
+
+type CachedSpeech = string | { text: string; complete?: boolean };
+
+export interface SpeechMessageCache {
+ get(key: string): Promise<CachedSpeech | null | undefined>;
+ set(key: string, value: CachedSpeech, ttl?: number): unknown;
+}
+
+/**
+ * Polls a message for the next text to stream as speech. Each call returns the chunks
+ * finalized since the previous one, or an error string once the message stays missing or
+ * unchanged too long. Only the answer is spoken: reasoning never reaches the chunks.
+ * Must be created within the request's tenant scope, which the cache key captures.
+ */
+export function createSpeechChunkProcessor({
+ user,
+ messageId,
+ cache,
+ getMessage,
+}: {
+ user: string;
+ messageId?: string;
+ cache: SpeechMessageCache;
+ getMessage: (filter: { user: string; messageId: string }) => Promise<TMessage | null>;
+}): () => Promise<SpeechChunk[] | string> {
+ if (!messageId) {
+ throw new Error('Message ID is required');
+ }
+
+ let notFoundCount = 0;
+ let noChangeCount = 0;
+ let processedText = '';
+ const cacheKey = scopedCacheKey(messageId);
+
+ return async function processChunks() {
+ if (notFoundCount >= MAX_NOT_FOUND_COUNT) {
+ return `Message not found after ${MAX_NOT_FOUND_COUNT} attempts`;
+ }
+
+ if (noChangeCount >= MAX_NO_CHANGE_COUNT) {
+ return `No change in message after ${MAX_NO_CHANGE_COUNT} attempts`;
+ }
+
+ const message: CachedSpeech | TMessage | null =
+ (await cache.get(cacheKey)) || (await getMessage({ user, messageId }));
+
+ if (!message) {
+ notFoundCount++;
+ return [];
+ }
+
+ const text = typeof message === 'string' ? message : getSpeechText(message);
+ cache.set(cacheKey, { text, complete: true }, Time.FIVE_MINUTES);
+ const complete =
+ typeof message !== 'string' && (!('complete' in message) || message.complete !== false);
+
+ if (text === processedText) {
+ noChangeCount++;
+ }
+
+ const remainingText = text.slice(processedText.length);
+ const chunks: SpeechChunk[] = [];
+
+ if (!complete && remainingText.length >= 20) {
+ const separatorIndex = findLastSeparatorIndex(remainingText);
+ if (separatorIndex !== -1) {
+ const chunkText = remainingText.slice(0, separatorIndex + 1);
+ chunks.push({ text: chunkText, isFinished: false });
+ processedText += chunkText;
+ } else {
+ chunks.push({ text: remainingText, isFinished: false });
+ processedText = text;
+ }
+ } else if (complete && remainingText.trim().length > 0) {
+ chunks.push({ text: remainingText.trim(), isFinished: true });
+ processedText = text;
+ }
+
+ return chunks;
+ };
+}
diff --git a/packages/data-provider/specs/parsers.spec.ts b/packages/data-provider/specs/parsers.spec.ts
index c1cf132f5d..392477f72c 100644
--- a/packages/data-provider/specs/parsers.spec.ts
+++ b/packages/data-provider/specs/parsers.spec.ts
@@ -1,6 +1,8 @@
import {
parseConvo,
+ getSpeechText,
parseTextParts,
+ parseThinkingContent,
parseCompactConvo,
replaceSpecialVars,
getEphemeralSender,
@@ -596,6 +598,74 @@ describe('parseCompactConvo - defaultParamsEndpoint', () => {
});
});
+describe('getSpeechText', () => {
+ test('speaks text parts and skips think parts', () => {
+ const content: TMessageContentParts[] = [
+ { type: ContentTypes.THINK, think: 'internal reasoning' },
+ { type: ContentTypes.TEXT, text: 'First.' },
+ { type: ContentTypes.THINK, think: 'more reasoning' },
+ { type: ContentTypes.TEXT, text: 'Second.' },
+ ];
+ expect(getSpeechText({ content, text: 'ignored' })).toBe('First. Second.');
+ });
+
+ test('is empty for a response that only reasoned', () => {
+ const content: TMessageContentParts[] = [{ type: ContentTypes.THINK, think: 'reasoning' }];
+ expect(getSpeechText({ content })).toBe('');
+ });
+
+ test('never speaks steer parts', () => {
+ const content: TMessageContentParts[] = [
+ { type: ContentTypes.STEER, steer: 'user words' } as TMessageContentParts,
+ { type: ContentTypes.TEXT, text: 'answer' },
+ ];
+ expect(getSpeechText({ content })).toBe('answer');
+ });
+
+ test('falls back to plain text when there are no content parts', () => {
+ expect(getSpeechText({ content: [], text: 'plain answer' })).toBe('plain answer');
+ expect(getSpeechText({ text: 'plain answer' })).toBe('plain answer');
+ expect(getSpeechText({ content: null, text: null })).toBe('');
+ });
+
+ test('speaks string content, as imported messages may carry it', () => {
+ const content = ':::thinking\nWork it out.\n:::\nThe answer is 4.';
+ expect(getSpeechText({ content, text: '' })).toBe('The answer is 4.');
+ });
+
+ test('falls back to text when string content is empty', () => {
+ expect(getSpeechText({ content: '', text: 'stored answer' })).toBe('stored answer');
+ });
+
+ test('strips a legacy thinking block from plain text', () => {
+ const text = ':::thinking\nWork it out.\n:::\nThe answer is 4.';
+ expect(getSpeechText({ text })).toBe('The answer is 4.');
+ });
+
+ test('speaks a repeated thinking block, which the message view shows as answer text', () => {
+ const text = ':::thinking\na\n:::\nAnswer 1\n:::thinking\nb\n:::\nAnswer 2';
+ expect(getSpeechText({ text })).toBe(parseThinkingContent(text).regularContent);
+ expect(getSpeechText({ text })).toBe('Answer 1\n:::thinking\nb\n:::\nAnswer 2');
+ });
+
+ test('trims structured answers so whitespace alone is not speakable', () => {
+ const content: TMessageContentParts[] = [
+ { type: ContentTypes.THINK, think: 'reasoning' },
+ { type: ContentTypes.TEXT, text: '\n' },
+ ];
+ expect(getSpeechText({ content })).toBe('');
+ });
+
+ test('speaks an unmatched marker, which the UI shows as answer text', () => {
+ const text = 'Wrap reasoning in a :::thinking block.';
+ expect(getSpeechText({ text })).toBe(text);
+ });
+
+ test('keeps plain text that merely uses colons', () => {
+ expect(getSpeechText({ text: 'Ratio is 3:1 ::: done' })).toBe('Ratio is 3:1 ::: done');
+ });
+});
+
describe('parseTextParts', () => {
test('should concatenate text parts', () => {
const parts: TMessageContentParts[] = [
diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.ts
index bf9ad180e7..a423fe455d 100644
--- a/packages/data-provider/src/parsers.ts
+++ b/packages/data-provider/src/parsers.ts
@@ -416,6 +416,40 @@ export function parseTextParts(
return result;
}
+const thinkingBlockPattern = /:::thinking([\s\S]*?):::/;
+
+/**
+ * Splits legacy plain-text reasoning from the answer exactly as the message view renders it:
+ * the first `:::thinking ... :::` block becomes the collapsible Thoughts, the rest is the answer.
+ */
+export function parseThinkingContent(text: string): {
+ thinkingContent: string;
+ regularContent: string;
+} {
+ const thinkingMatch = text.match(thinkingBlockPattern);
+ return {
+ thinkingContent: thinkingMatch ? thinkingMatch[1].trim() : '',
+ regularContent: thinkingMatch ? text.replace(thinkingBlockPattern, '').trim() : text,
+ };
+}
+
+/**
+ * The text Read Aloud speaks for a message: the answer the message view shows, never its
+ * reasoning. Covers both reasoning shapes the UI collapses into Thoughts: `think` content parts
+ * and the legacy `:::thinking ... :::` block in plain text.
+ */
+export function getSpeechText(message: {
+ content?: Array<a.TMessageContentParts | undefined> | string | null;
+ text?: string | null;
+}): string {
+ if (Array.isArray(message.content) && message.content.length > 0) {
+ return parseTextParts(message.content, true).trim();
+ }
+ const text =
+ typeof message.content === 'string' && message.content ? message.content : message.text;
+ return parseThinkingContent(text ?? '').regularContent.trim();
+}
+
export const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n', '```'];
export function findLastSeparatorIndex(text: string, separators = SEPARATORS): number {