Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔇 fix: Keep Reasoning Out of Read Aloud (#16790)
Marco Beretta9068f0e92215
diff --git a/api/server/services/Files/Audio/streamAudio.js b/api/server/services/Files/Audio/streamAudio.jsindex 7120399b5e..ab7c7d9c3c 100644--- a/api/server/services/Files/Audio/streamAudio.js+++ b/api/server/services/Files/Audio/streamAudio.js@@ -1,11 +1,5 @@-const { scopedCacheKey } = require('@librechat/data-schemas');-const {- Time,- CacheKeys,- SEPARATORS,- parseTextParts,- findLastSeparatorIndex,-} = require('librechat-data-provider');+const { createSpeechChunkProcessor } = require('@librechat/api');+const { CacheKeys, SEPARATORS, findLastSeparatorIndex } = require('librechat-data-provider');const { getLogStores } = require('~/cache');const { getMessage } = require('~/models');@@ -51,88 +45,18 @@ function getRandomVoiceId(voiceIds) {* @property {string[]} normalizedAlignment.chars*/-const MAX_NOT_FOUND_COUNT = 6;-const MAX_NO_CHANGE_COUNT = 10;-/*** @param {string} user* @param {string} messageId- * @returns {() => Promise<{ text: string, isFinished: boolean }[]>}+ * @returns {() => Promise<{ text: string, isFinished: boolean }[] | string>}*/function createChunkProcessor(user, messageId) {- let notFoundCount = 0;- let noChangeCount = 0;- let processedText = '';- if (!messageId) {- throw new Error('Message ID is required');- }-- const messageCache = getLogStores(CacheKeys.MESSAGES);- // Captured at creation time — must be called within an active request ALS scope- const cacheKey = scopedCacheKey(messageId);-- /**- * @returns {Promise<{ text: string, isFinished: boolean }[] | string>}- */- async function processChunks() {- if (notFoundCount >= MAX_NOT_FOUND_COUNT) {- return `Message not found after ${MAX_NOT_FOUND_COUNT} attempts`;- }-- if (noChangeCount >= MAX_NO_CHANGE_COUNT) {- return `No change in message after ${MAX_NO_CHANGE_COUNT} attempts`;- }-- /** @type { string | { text: string; complete: boolean } } */- let message = await messageCache.get(cacheKey);- if (!message) {- message = await getMessage({ user, messageId });- }-- if (!message) {- notFoundCount++;- return [];- } else {- const text = message.content?.length > 0 ? parseTextParts(message.content) : message.text;- messageCache.set(- cacheKey,- {- text,- complete: true,- },- Time.FIVE_MINUTES,- );- }-- const text = typeof message === 'string' ? message : message.text;- const complete = typeof message === 'string' ? false : (message.complete ?? true);-- if (text === processedText) {- noChangeCount++;- }-- const remainingText = text.slice(processedText.length);- const chunks = [];-- if (!complete && remainingText.length >= 20) {- const separatorIndex = findLastSeparatorIndex(remainingText);- if (separatorIndex !== -1) {- const chunkText = remainingText.slice(0, separatorIndex + 1);- chunks.push({ text: chunkText, isFinished: false });- processedText += chunkText;- } else {- chunks.push({ text: remainingText, isFinished: false });- processedText = text;- }- } else if (complete && remainingText.trim().length > 0) {- chunks.push({ text: remainingText.trim(), isFinished: true });- processedText = text;- }-- return chunks;- }-- return processChunks;+ return createSpeechChunkProcessor({+ user,+ messageId,+ cache: getLogStores(CacheKeys.MESSAGES),+ getMessage,+ });}/**diff --git a/api/server/services/Files/Audio/streamAudio.spec.js b/api/server/services/Files/Audio/streamAudio.spec.jsindex 977d8730aa..021e8373de 100644--- a/api/server/services/Files/Audio/streamAudio.spec.js+++ b/api/server/services/Files/Audio/streamAudio.spec.js@@ -1,4 +1,4 @@-const { createChunkProcessor, splitTextIntoChunks } = require('./streamAudio');+const { splitTextIntoChunks } = require('./streamAudio');jest.mock('keyv');@@ -55,102 +55,6 @@ jest.mock('~/cache/getLogStores', () => {});});-describe('processChunks', () => {- let processChunks;- let mockMessageCache;-- beforeEach(() => {- jest.resetAllMocks();- mockMessageCache = {- get: jest.fn(),- set: jest.fn(),- };- require('~/cache/getLogStores').mockReturnValue(mockMessageCache);- processChunks = createChunkProcessor('userId', 'message-id');- });-- it('should return an empty array when the message is not found', async () => {- mockMessageCache.get.mockResolvedValueOnce(null);-- const result = await processChunks();-- expect(result).toEqual([]);- expect(mockMessageCache.get).toHaveBeenCalledWith('message-id');- });-- it('should return an error message after MAX_NOT_FOUND_COUNT attempts', async () => {- mockMessageCache.get.mockResolvedValue(null);-- for (let i = 0; i < 6; i++) {- await processChunks();- }- const result = await processChunks();-- expect(result).toBe('Message not found after 6 attempts');- });-- it('should return chunks for an incomplete message with separators', async () => {- const messageText = 'This is a long message. It should be split into chunks. Lol hi mom';- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: false });-- const result = await processChunks();-- expect(result).toEqual([- { text: 'This is a long message. It should be split into chunks.', isFinished: false },- ]);- });-- it('should return chunks for an incomplete message without separators', async () => {- const messageText = 'This is a long message without separators hello there my friend';- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: false });-- const result = await processChunks();-- expect(result).toEqual([{ text: messageText, isFinished: false }]);- });-- it('should return the remaining text as a chunk for a complete message', async () => {- const messageText = 'This is a finished message.';- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: true });-- const result = await processChunks();-- expect(result).toEqual([{ text: messageText, isFinished: true }]);- });-- it('should return an empty array for a complete message with no remaining text', async () => {- const messageText = 'This is a finished message.';- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: true });-- await processChunks();- mockMessageCache.get.mockResolvedValueOnce({ text: messageText, complete: true });- const result = await processChunks();-- expect(result).toEqual([]);- });-- it('should return an error message after MAX_NO_CHANGE_COUNT attempts with no change', async () => {- const messageText = 'This is a message that does not change.';- mockMessageCache.get.mockResolvedValue({ text: messageText, complete: false });-- for (let i = 0; i < 11; i++) {- await processChunks();- }- const result = await processChunks();-- expect(result).toBe('No change in message after 10 attempts');- });-- it('should handle string messages as incomplete', async () => {- const messageText = 'This is a message as a string.';- mockMessageCache.get.mockResolvedValueOnce(messageText);-- const result = await processChunks();-- expect(result).toEqual([{ text: messageText, isFinished: false }]);- });-});-describe('splitTextIntoChunks', () => {test('splits text into chunks of specified size with default separators', () => {const text = 'This is a test. This is only a test! Make sure it works properly? Okay.';diff --git a/client/src/components/Chat/Input/BrowserAudio.tsx b/client/src/components/Chat/Input/BrowserAudio.tsxindex 262310f7b2..1da7af5073 100644--- a/client/src/components/Chat/Input/BrowserAudio.tsx+++ b/client/src/components/Chat/Input/BrowserAudio.tsx@@ -1,7 +1,7 @@import { useEffect } from 'react';import { useSetRecoilState } from 'recoil';import { useParams } from 'react-router-dom';-import { parseTextParts } from 'librechat-data-provider';+import { getSpeechText } from 'librechat-data-provider';import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';import useAutoplayTrigger from '~/hooks/Audio/useAutoplayTrigger';import { logger } from '~/utils';@@ -36,10 +36,7 @@ export default function BrowserAudio({ index = 0 }) {return;}- const text =- Array.isArray(latestMessage.content) && latestMessage.content.length > 0- ? parseTextParts(latestMessage.content)- : (latestMessage.text ?? '');+ const text = getSpeechText(latestMessage);if (!text) {return;diff --git a/client/src/components/Chat/Input/StreamAudio.tsx b/client/src/components/Chat/Input/StreamAudio.tsxindex d7e788da46..5db4b9c767 100644--- a/client/src/components/Chat/Input/StreamAudio.tsx+++ b/client/src/components/Chat/Input/StreamAudio.tsx@@ -1,7 +1,7 @@import { useEffect, useCallback } from 'react';import { useParams } from 'react-router-dom';-import { QueryKeys } from 'librechat-data-provider';import { useQueryClient } from '@tanstack/react-query';+import { QueryKeys, getSpeechText } from 'librechat-data-provider';import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';import type { TMessage } from 'librechat-data-provider';import {@@ -66,7 +66,9 @@ export default function StreamAudio({ index = 0 }) {setGlobalAudioURL(null);}- let cacheKey = latestMessage?.text ?? '';+ /** Keyed by the spoken text, as manual playback is, so audio cached from an+ * unfiltered message (reasoning included) is never replayed. */+ let cacheKey = latestMessage ? getSpeechText(latestMessage) : '';const cache = await caches.open('tts-responses');const cachedResponse = await cache.match(cacheKey);@@ -139,7 +141,7 @@ export default function StreamAudio({ index = 0 }) {const targetMessage = latestMessages.find((msg) => msg.messageId === latestMessage?.messageId,);- cacheKey = targetMessage?.text ?? '';+ cacheKey = targetMessage ? getSpeechText(targetMessage) : '';if (!cacheKey) {logger.warn('Cache key not found, skipping audio cache');} else {diff --git a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsxindex 3dcb631a07..b14a53ae6a 100644--- a/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx+++ b/client/src/components/Chat/Input/__tests__/BrowserAudio.spec.tsx@@ -1,9 +1,9 @@import React from 'react';import { RecoilRoot } from 'recoil';import { render, waitFor } from '@testing-library/react';-import { Constants, QueryKeys } from 'librechat-data-provider';import { MemoryRouter, Routes, Route } from 'react-router-dom';import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider';import type { TMessage, TConversation } from 'librechat-data-provider';import BrowserAudio from '../BrowserAudio';import store from '~/store';@@ -108,6 +108,23 @@ describe('BrowserAudio autoplay', () => {await waitFor(() => expect(spoken).toEqual([responseText]));});+ it('speaks the answer without the reasoning that preceded it', async () => {+ renderBrowserAudio({+ messages: [+ {+ ...assistantMessage,+ text: '',+ content: [+ { type: ContentTypes.THINK, think: 'Recalling European capitals.' },+ { type: ContentTypes.TEXT, text: responseText },+ ],+ },+ ],+ });++ await waitFor(() => expect(spoken).toEqual([responseText]));+ });+it('does not speak while the run is still submitting', async () => {renderBrowserAudio({ isSubmitting: true });await settle();@@ -130,6 +147,22 @@ describe('BrowserAudio autoplay', () => {expect(spoken).toEqual([]);});+ /** A turn stopped mid-reasoning persists its reasoning as `text` beside the parts. */+ it('does not autoplay a stopped turn that only reasoned', async () => {+ renderBrowserAudio({+ messages: [+ {+ ...assistantMessage,+ text: 'Recalling European capitals.',+ content: [{ type: ContentTypes.THINK, think: 'Recalling European capitals.' }],+ },+ ],+ });+ await settle();++ expect(spoken).toEqual([]);+ });+it('does not speak the user message back to them', async () => {const userMessage = { ...assistantMessage, isCreatedByUser: true } as TMessage;renderBrowserAudio({ messages: [userMessage] });diff --git a/client/src/components/Chat/Input/__tests__/StreamAudio.spec.tsx b/client/src/components/Chat/Input/__tests__/StreamAudio.spec.tsxnew file mode 100644index 0000000000..3ddf54f75a--- /dev/null+++ b/client/src/components/Chat/Input/__tests__/StreamAudio.spec.tsx@@ -0,0 +1,64 @@+import React from 'react';+import { RecoilRoot } from 'recoil';+import { render, waitFor } from '@testing-library/react';+import { MemoryRouter, Routes, Route } from 'react-router-dom';+import { Constants, QueryKeys } from 'librechat-data-provider';+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import type { TMessage, TConversation } from 'librechat-data-provider';+import StreamAudio from '../StreamAudio';+import store from '~/store';++jest.mock('~/hooks', () => ({ useAuthContext: () => ({ token: 'token' }) }));++const conversationId = 'convo-1';++const match = jest.fn(async (_key: string) => ({ blob: async () => new Blob() }));++describe('StreamAudio cache', () => {+ beforeAll(() => {+ Object.defineProperty(global, 'caches', {+ writable: true,+ configurable: true,+ value: { open: async () => ({ match, put: jest.fn() }) },+ });+ URL.createObjectURL = jest.fn(() => 'blob:audio');+ URL.revokeObjectURL = jest.fn();+ });++ /** Audio cached before reasoning was filtered is keyed by the full text; looking up the+ * spoken answer instead means that recording is never replayed. */+ it('looks up cached audio by the spoken answer, not the stored text', async () => {+ const message = {+ messageId: 'assistant-1',+ conversationId,+ parentMessageId: Constants.NO_PARENT,+ isCreatedByUser: false,+ text: ':::thinking\nRecalling European capitals.\n:::\nThe capital of Germany is Berlin.',+ } as TMessage;+ const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });+ queryClient.setQueryData([QueryKeys.messages, conversationId], [message]);++ render(+ <QueryClientProvider client={queryClient}>+ <MemoryRouter initialEntries={[`/c/${conversationId}`]}>+ <RecoilRoot+ initializeState={({ set }) => {+ set(store.automaticPlayback, true);+ set(store.conversationByIndex(0), { conversationId } as TConversation);+ set(store.activeRunFamily(0), 'run-1');+ set(store.audioRunFamily(0), null);+ set(store.isSubmittingFamily(0), false);+ }}+ >+ <Routes>+ <Route path="/c/:conversationId" element={<StreamAudio index={0} />} />+ </Routes>+ </RecoilRoot>+ </MemoryRouter>+ </QueryClientProvider>,+ );++ await waitFor(() => expect(match).toHaveBeenCalled());+ expect(match).toHaveBeenCalledWith('The capital of Germany is Berlin.');+ });+});diff --git a/client/src/components/Chat/Messages/Content/MessageContent.tsx b/client/src/components/Chat/Messages/Content/MessageContent.tsxindex 6cea94d3ac..4502cef6ad 100644--- a/client/src/components/Chat/Messages/Content/MessageContent.tsx+++ b/client/src/components/Chat/Messages/Content/MessageContent.tsx@@ -1,7 +1,7 @@import { memo, Suspense, useMemo } from 'react';import { useRecoilValue } from 'recoil';-import { Constants } from 'librechat-data-provider';import { Alert, DelayedRender } from '@librechat/client';+import { Constants, parseThinkingContent } from 'librechat-data-provider';import type { TMessage } from 'librechat-data-provider';import type { TMessageContentProps, TDisplayProps } from '~/common';import useSmoothStreaming from '~/hooks/Messages/useSmoothStreaming';@@ -23,14 +23,6 @@ const ERROR_CONNECTION_TEXT = 'Error connecting to server, try refreshing the paconst DELAYED_ERROR_TIMEOUT = 5500;const UNFINISHED_DELAY = 250;-const parseThinkingContent = (text: string) => {- const thinkingMatch = text.match(/:::thinking([\s\S]*?):::/);- return {- thinkingContent: thinkingMatch ? thinkingMatch[1].trim() : '',- regularContent: thinkingMatch ? text.replace(/:::thinking[\s\S]*?:::/, '').trim() : text,- };-};-const LoadingFallback = () => (<div className="mb-[0.625rem]"><EmptyText underHeaderIcon />diff --git a/client/src/components/Chat/Messages/HoverButtons.tsx b/client/src/components/Chat/Messages/HoverButtons.tsxindex d608ea7747..225daa9bf3 100644--- a/client/src/components/Chat/Messages/HoverButtons.tsx+++ b/client/src/components/Chat/Messages/HoverButtons.tsx@@ -2,7 +2,7 @@ import React, { useState, useMemo, useCallback, memo } from 'react';import { Copy, Check } from 'lucide';import { useAtomValue } from 'jotai';import { useRecoilState } from 'recoil';-import { findMessageById, isUserInitiatedCompaction } from 'librechat-data-provider';+import { getSpeechText, findMessageById, isUserInitiatedCompaction } from 'librechat-data-provider';import {Button,EditIcon,@@ -55,38 +55,6 @@ type HoverButtonProps = {disabled?: boolean;};-const extractMessageContent = (message: TMessage): string => {- if (typeof message.content === 'string') {- return message.content;- }-- if (Array.isArray(message.content)) {- return message.content- .map((part) => {- if (part == null) {- return '';- }- if (typeof part === 'string') {- return part;- }- if ('text' in part) {- return part.text || '';- }- if ('think' in part) {- const think = part.think;- if (typeof think === 'string') {- return think;- }- return think && 'text' in think ? think.text || '' : '';- }- return '';- })- .join('');- }-- return message.text || '';-};-const HoverButton = memo(({id,@@ -226,16 +194,17 @@ const HoverButtons = ({};const handleCopy = () => copyToClipboard(setIsCopied);+ const speechText = getSpeechText(message);return (<div className="group visible flex justify-center gap-0.5 self-end focus-within:outline-hidden lg:justify-start">{/* Text to Speech */}- {TextToSpeech && !error && !isActiveStreamingMessage && (+ {TextToSpeech && !error && !isActiveStreamingMessage && speechText.length > 0 && (<MessageAudioindex={index}isLast={isLast}messageId={message.messageId}- content={extractMessageContent(message)}+ content={speechText}renderButton={(props) => (<HoverButtononClick={props.onClick}diff --git a/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx b/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsxindex 55cd394c0f..4dfdc6a49a 100644--- a/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx+++ b/client/src/components/Chat/Messages/__tests__/HoverButtons.spec.tsx@@ -1,7 +1,7 @@import React from 'react';import { MemoryRouter } from 'react-router-dom';import { RecoilRoot, type MutableSnapshot } from 'recoil';-import { act, render, screen } from '@testing-library/react';+import { act, fireEvent, render, screen } from '@testing-library/react';import { QueryClient, QueryClientProvider } from '@tanstack/react-query';import {Constants,@@ -42,6 +42,7 @@ function renderHoverButtons({getCanCopy = () => hasCopyableText({ text: message.text, content: message.content }),handleFeedback,thread,+ voice,}: {isSubmitting: boolean;message?: TMessage;@@ -53,12 +54,20 @@ function renderHoverButtons({/** The rows the hover controls resolve the message's parent from. Omitted, the* thread is unavailable, as on a search row. */thread?: TMessage[];+ /** Enables Read Aloud on the browser engine with this voice selected. */+ voice?: string;}) {const queryClient = new QueryClient({defaultOptions: { queries: { retry: false } },});- const initializeState = ({ set }: MutableSnapshot) => set(store.textToSpeech, false);+ const initializeState = ({ set }: MutableSnapshot) => {+ set(store.textToSpeech, voice != null);+ if (voice != null) {+ set(store.voice, voice);+ set(store.speechSettingsInitialized, true);+ }+ };const { container } = render(<QueryClientProvider client={queryClient}>@@ -639,3 +648,87 @@ describe('HoverButtons feedback affordance', () => {expect(screen.getByTestId('copy-response-button')).toBeInTheDocument();});});++describe('HoverButtons read aloud', () => {+ const voiceName = 'Test Voice';+ const spoken: string[] = [];++ class FakeSpeechSynthesisUtterance {+ public voice: SpeechSynthesisVoice | null = null;+ public onend: (() => void) | null = null;+ public onerror: (() => void) | null = null;+ constructor(public text: string) {}+ }++ beforeAll(() => {+ /** jsdom lacks it; the message's audio element revokes its source on unmount. */+ URL.revokeObjectURL = jest.fn();+ Object.defineProperty(window, 'speechSynthesis', {+ writable: true,+ configurable: true,+ value: {+ getVoices: () => [{ name: voiceName, localService: true }],+ addEventListener: () => undefined,+ speak: (utterance: FakeSpeechSynthesisUtterance) => spoken.push(utterance.text),+ cancel: () => undefined,+ },+ });+ Object.defineProperty(global, 'SpeechSynthesisUtterance', {+ writable: true,+ configurable: true,+ value: FakeSpeechSynthesisUtterance,+ });+ });++ beforeEach(() => {+ spoken.length = 0;+ });++ const readAloud = (message: Partial<TMessage>) => {+ renderHoverButtons({+ isSubmitting: false,+ message: { ...userMessage, messageId: 'assistant-1', isCreatedByUser: false, ...message },+ isLast: true,+ latestMessageId: 'assistant-1',+ voice: voiceName,+ });+ fireEvent.click(screen.getByTestId('read-aloud-button'));+ };++ it('speaks the answer without the reasoning parts before it', () => {+ readAloud({+ text: '',+ content: [+ { type: ContentTypes.THINK, think: 'Let me work out 2 + 2 first.' },+ { type: ContentTypes.TEXT, text: 'The answer is 4.' },+ ],+ });++ expect(spoken).toEqual(['The answer is 4.']);+ });++ it('offers no read aloud for a response that only reasoned', () => {+ renderHoverButtons({+ isSubmitting: false,+ message: {+ ...userMessage,+ messageId: 'assistant-1',+ isCreatedByUser: false,+ text: 'Let me work out 2 + 2 first.',+ content: [{ type: ContentTypes.THINK, think: 'Let me work out 2 + 2 first.' }],+ },+ isLast: true,+ latestMessageId: 'assistant-1',+ voice: voiceName,+ });++ expect(screen.queryByTestId('read-aloud-button')).toBeNull();+ expect(screen.getByTestId('copy-response-button')).toBeInTheDocument();+ });++ it('speaks the answer without a legacy thinking block in plain text', () => {+ readAloud({ text: ':::thinking\nLet me work out 2 + 2 first.\n:::\nThe answer is 4.' });++ expect(spoken).toEqual(['The answer is 4.']);+ });+});diff --git a/client/src/hooks/Audio/__tests__/useAutoplayTrigger.spec.tsx b/client/src/hooks/Audio/__tests__/useAutoplayTrigger.spec.tsxnew file mode 100644index 0000000000..2528440e0d--- /dev/null+++ b/client/src/hooks/Audio/__tests__/useAutoplayTrigger.spec.tsx@@ -0,0 +1,63 @@+import React from 'react';+import { RecoilRoot } from 'recoil';+import { renderHook } from '@testing-library/react';+import { MemoryRouter, Routes, Route } from 'react-router-dom';+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import { Constants, QueryKeys, ContentTypes } from 'librechat-data-provider';+import type { TMessage, TConversation } from 'librechat-data-provider';+import useAutoplayTrigger from '../useAutoplayTrigger';+import store from '~/store';++const conversationId = 'convo-1';++const assistantMessage = {+ messageId: 'assistant-1',+ conversationId,+ parentMessageId: Constants.NO_PARENT,+ isCreatedByUser: false,+ text: 'The capital of Germany is Berlin.',+} as TMessage;++const renderTrigger = (message: TMessage) => {+ const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });+ queryClient.setQueryData([QueryKeys.messages, conversationId], [message]);++ const wrapper = ({ children }: { children: React.ReactNode }) => (+ <QueryClientProvider client={queryClient}>+ <MemoryRouter initialEntries={[`/c/${conversationId}`]}>+ <RecoilRoot+ initializeState={({ set }) => {+ set(store.conversationByIndex(0), { conversationId } as TConversation);+ set(store.activeRunFamily(0), 'run-1');+ set(store.audioRunFamily(0), null);+ set(store.isSubmittingFamily(0), false);+ }}+ >+ <Routes>+ <Route path="/c/:conversationId" element={<>{children}</>} />+ </Routes>+ </RecoilRoot>+ </MemoryRouter>+ </QueryClientProvider>+ );++ return renderHook(() => useAutoplayTrigger(0), { wrapper }).result;+};++describe('useAutoplayTrigger', () => {+ it('plays a finalized answer', () => {+ expect(renderTrigger(assistantMessage).current.shouldPlay).toBe(true);+ });++ /** A turn stopped mid-reasoning persists its reasoning as `text` beside the parts; playing+ * it would open a server stream with nothing to speak. */+ it('does not play a stopped turn that only reasoned', () => {+ const result = renderTrigger({+ ...assistantMessage,+ text: 'Recalling European capitals.',+ content: [{ type: ContentTypes.THINK, think: 'Recalling European capitals.' }],+ });++ expect(result.current.shouldPlay).toBe(false);+ });+});diff --git a/client/src/hooks/Audio/useAutoplayTrigger.ts b/client/src/hooks/Audio/useAutoplayTrigger.tsindex ec257808f8..67049d58cc 100644--- a/client/src/hooks/Audio/useAutoplayTrigger.ts+++ b/client/src/hooks/Audio/useAutoplayTrigger.ts@@ -1,7 +1,7 @@import { useRecoilValue } from 'recoil';+import { getSpeechText } from 'librechat-data-provider';import type { TMessage } from 'librechat-data-provider';import { useLatestMessage } from '~/hooks/Messages/useLatestMessage';-import { getLatestText } from '~/utils';import store from '~/store';export type TAutoplayTrigger = {@@ -14,7 +14,7 @@ export type TAutoplayTrigger = {/*** Shared "Autoplay Latest Message" gate so every TTS engine autoplays on identical terms:* the run must be finished, the branch tail must be a persisted assistant message carrying- * text, and its run must not have been played already.+ * speakable text (reasoning alone is not), and its run must not have been played already.*/export default function useAutoplayTrigger(index: string | number = 0): TAutoplayTrigger {const activeRunId = useRecoilValue(store.activeRunFamily(index));@@ -26,7 +26,7 @@ export default function useAutoplayTrigger(index: string | number = 0): TAutopla!isSubmitting &&latestMessage &&latestMessage.isCreatedByUser !== true &&- getLatestText(latestMessage) &&+ getSpeechText(latestMessage) &&latestMessage.messageId &&!latestMessage.messageId.includes('_') &&activeRunId != null &&diff --git a/client/src/hooks/Audio/useTTSBrowser.ts b/client/src/hooks/Audio/useTTSBrowser.tsindex 9edd9a06ec..215705565c 100644--- a/client/src/hooks/Audio/useTTSBrowser.ts+++ b/client/src/hooks/Audio/useTTSBrowser.ts@@ -61,7 +61,9 @@ const useTTSBrowser = (props?: TUseTextToSpeech) => {if (isMouseDownRef.current) {const messageContent = content ?? '';const parsedMessage =- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);+ typeof messageContent === 'string'+ ? messageContent+ : parseTextParts(messageContent, true);generateSpeech(parsedMessage);}}, 1000);@@ -81,7 +83,7 @@ const useTTSBrowser = (props?: TUseTextToSpeech) => {} else {const messageContent = content ?? '';const parsedMessage =- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent, true);generateSpeech(parsedMessage);}};diff --git a/client/src/hooks/Audio/useTTSExternal.ts b/client/src/hooks/Audio/useTTSExternal.tsindex d296abf4a4..634e877489 100644--- a/client/src/hooks/Audio/useTTSExternal.ts+++ b/client/src/hooks/Audio/useTTSExternal.ts@@ -62,7 +62,9 @@ const useTTSExternal = (props?: TUseTextToSpeech) => {if (isMouseDownRef.current) {const messageContent = content ?? '';const parsedMessage =- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);+ typeof messageContent === 'string'+ ? messageContent+ : parseTextParts(messageContent, true);generateSpeech(parsedMessage, false);}}, 1000);@@ -82,7 +84,7 @@ const useTTSExternal = (props?: TUseTextToSpeech) => {} else {const messageContent = content ?? '';const parsedMessage =- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent, true);generateSpeech(parsedMessage, false);}};diff --git a/client/src/hooks/Input/useTextToSpeech.ts b/client/src/hooks/Input/useTextToSpeech.tsindex 12da8ef65c..c5eff56a5b 100644--- a/client/src/hooks/Input/useTextToSpeech.ts+++ b/client/src/hooks/Input/useTextToSpeech.ts@@ -1,5 +1,5 @@-import { useRecoilState, useRecoilValue } from 'recoil';import { useRef, useMemo, useEffect, useState } from 'react';+import { useRecoilState, useRecoilValue } from 'recoil';import { parseTextParts } from 'librechat-data-provider';import type { TMessageContentParts } from 'librechat-data-provider';import type { Option } from '~/common';@@ -120,7 +120,9 @@ const useTextToSpeech = (props?: TUseTextToSpeech) => {if (isMouseDownRef.current) {const messageContent = content ?? '';const parsedMessage =- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);+ typeof messageContent === 'string'+ ? messageContent+ : parseTextParts(messageContent, true);generateSpeech(parsedMessage, false);}}, 1000);@@ -140,7 +142,7 @@ const useTextToSpeech = (props?: TUseTextToSpeech) => {} else {const messageContent = content ?? '';const parsedMessage =- typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent, true);generateSpeech(parsedMessage, false);}};diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.tsindex fb20a690ed..66226ed4b4 100644--- a/packages/api/src/files/index.ts+++ b/packages/api/src/files/index.ts@@ -22,6 +22,7 @@ export * from './rag/context';export * from './rag/search';export * from './regexEngine';export * from './retention';+export * from './speech';export * from './sse';export * from './sweep';export * from './usage';diff --git a/packages/api/src/files/speech.spec.ts b/packages/api/src/files/speech.spec.tsnew file mode 100644index 0000000000..b939bac4da--- /dev/null+++ b/packages/api/src/files/speech.spec.ts@@ -0,0 +1,116 @@+import { ContentTypes } from 'librechat-data-provider';+import type { TMessage } from 'librechat-data-provider';+import type { SpeechMessageCache } from './speech';+import { createSpeechChunkProcessor } from './speech';++const messageId = 'message-id';++function setup(stored: TMessage | null = null) {+ const entries = new Map<string, Parameters<SpeechMessageCache['set']>[1]>();+ const cache = {+ get: jest.fn(async (key: string) => entries.get(key)),+ set: jest.fn((key: string, value: Parameters<SpeechMessageCache['set']>[1]) =>+ entries.set(key, value),+ ),+ };+ const getMessage = jest.fn(async () => stored);+ const processChunks = createSpeechChunkProcessor({+ user: 'user-id',+ messageId,+ cache,+ getMessage,+ });+ return { cache, getMessage, processChunks };+}++describe('createSpeechChunkProcessor', () => {+ it('requires a message id', () => {+ expect(() =>+ createSpeechChunkProcessor({+ user: 'user-id',+ cache: { get: jest.fn(), set: jest.fn() },+ getMessage: jest.fn(),+ }),+ ).toThrow('Message ID is required');+ });++ it('returns no chunks while the message is not found', async () => {+ const { cache, getMessage, processChunks } = setup();++ expect(await processChunks()).toEqual([]);+ expect(cache.get).toHaveBeenCalledWith(messageId);+ expect(getMessage).toHaveBeenCalledWith({ user: 'user-id', messageId });+ });++ it('gives up after the message stays missing', async () => {+ const { processChunks } = setup();++ for (let i = 0; i < 6; i++) {+ await processChunks();+ }++ expect(await processChunks()).toBe('Message not found after 6 attempts');+ });++ it('splits an incomplete message at its last separator', async () => {+ const { cache, processChunks } = setup();+ cache.get.mockResolvedValueOnce({+ text: 'This is a long message. It should be split into chunks. Lol hi mom',+ complete: false,+ });++ expect(await processChunks()).toEqual([+ { text: 'This is a long message. It should be split into chunks.', isFinished: false },+ ]);+ });++ it('sends an incomplete message without separators whole', async () => {+ const text = 'This is a long message without separators hello there my friend';+ const { cache, processChunks } = setup();+ cache.get.mockResolvedValueOnce({ text, complete: false });++ expect(await processChunks()).toEqual([{ text, isFinished: false }]);+ });++ it('finishes with the remaining text of a complete message', async () => {+ const text = 'This is a finished message.';+ const { cache, processChunks } = setup();+ cache.get.mockResolvedValueOnce({ text, complete: true });++ expect(await processChunks()).toEqual([{ text, isFinished: true }]);+ expect(await processChunks()).toEqual([]);+ });++ it('gives up after the message stops changing', async () => {+ const { cache, processChunks } = setup();+ cache.get.mockResolvedValue({ text: 'This message does not change.', complete: false });++ for (let i = 0; i < 11; i++) {+ await processChunks();+ }++ expect(await processChunks()).toBe('No change in message after 10 attempts');+ });++ it('treats a cached string as incomplete', async () => {+ const text = 'This is a message as a string.';+ const { cache, processChunks } = setup();+ cache.get.mockResolvedValueOnce(text);++ expect(await processChunks()).toEqual([{ text, isFinished: false }]);+ });++ /** An aborted turn persists `text` with its reasoning alongside the content parts. */+ it('never speaks the reasoning of a stored message', async () => {+ const { processChunks } = setup({+ messageId,+ text: 'Let me work this out. The answer is 4.',+ content: [+ { type: ContentTypes.THINK, think: 'Let me work this out.' },+ { type: ContentTypes.TEXT, text: 'The answer is 4.' },+ ],+ } as TMessage);++ expect(await processChunks()).toEqual([{ text: 'The answer is 4.', isFinished: true }]);+ });+});diff --git a/packages/api/src/files/speech.ts b/packages/api/src/files/speech.tsnew file mode 100644index 0000000000..4b414be1e9--- /dev/null+++ b/packages/api/src/files/speech.ts@@ -0,0 +1,89 @@+import { scopedCacheKey } from '@librechat/data-schemas';+import { Time, getSpeechText, findLastSeparatorIndex } from 'librechat-data-provider';+import type { TMessage } from 'librechat-data-provider';++const MAX_NOT_FOUND_COUNT = 6;+const MAX_NO_CHANGE_COUNT = 10;++export type SpeechChunk = { text: string; isFinished: boolean };++type CachedSpeech = string | { text: string; complete?: boolean };++export interface SpeechMessageCache {+ get(key: string): Promise<CachedSpeech | null | undefined>;+ set(key: string, value: CachedSpeech, ttl?: number): unknown;+}++/**+ * Polls a message for the next text to stream as speech. Each call returns the chunks+ * finalized since the previous one, or an error string once the message stays missing or+ * unchanged too long. Only the answer is spoken: reasoning never reaches the chunks.+ * Must be created within the request's tenant scope, which the cache key captures.+ */+export function createSpeechChunkProcessor({+ user,+ messageId,+ cache,+ getMessage,+}: {+ user: string;+ messageId?: string;+ cache: SpeechMessageCache;+ getMessage: (filter: { user: string; messageId: string }) => Promise<TMessage | null>;+}): () => Promise<SpeechChunk[] | string> {+ if (!messageId) {+ throw new Error('Message ID is required');+ }++ let notFoundCount = 0;+ let noChangeCount = 0;+ let processedText = '';+ const cacheKey = scopedCacheKey(messageId);++ return async function processChunks() {+ if (notFoundCount >= MAX_NOT_FOUND_COUNT) {+ return `Message not found after ${MAX_NOT_FOUND_COUNT} attempts`;+ }++ if (noChangeCount >= MAX_NO_CHANGE_COUNT) {+ return `No change in message after ${MAX_NO_CHANGE_COUNT} attempts`;+ }++ const message: CachedSpeech | TMessage | null =+ (await cache.get(cacheKey)) || (await getMessage({ user, messageId }));++ if (!message) {+ notFoundCount++;+ return [];+ }++ const text = typeof message === 'string' ? message : getSpeechText(message);+ cache.set(cacheKey, { text, complete: true }, Time.FIVE_MINUTES);+ const complete =+ typeof message !== 'string' && (!('complete' in message) || message.complete !== false);++ if (text === processedText) {+ noChangeCount++;+ }++ const remainingText = text.slice(processedText.length);+ const chunks: SpeechChunk[] = [];++ if (!complete && remainingText.length >= 20) {+ const separatorIndex = findLastSeparatorIndex(remainingText);+ if (separatorIndex !== -1) {+ const chunkText = remainingText.slice(0, separatorIndex + 1);+ chunks.push({ text: chunkText, isFinished: false });+ processedText += chunkText;+ } else {+ chunks.push({ text: remainingText, isFinished: false });+ processedText = text;+ }+ } else if (complete && remainingText.trim().length > 0) {+ chunks.push({ text: remainingText.trim(), isFinished: true });+ processedText = text;+ }++ return chunks;+ };+}diff --git a/packages/data-provider/specs/parsers.spec.ts b/packages/data-provider/specs/parsers.spec.tsindex c1cf132f5d..392477f72c 100644--- a/packages/data-provider/specs/parsers.spec.ts+++ b/packages/data-provider/specs/parsers.spec.ts@@ -1,6 +1,8 @@import {parseConvo,+ getSpeechText,parseTextParts,+ parseThinkingContent,parseCompactConvo,replaceSpecialVars,getEphemeralSender,@@ -596,6 +598,74 @@ describe('parseCompactConvo - defaultParamsEndpoint', () => {});});+describe('getSpeechText', () => {+ test('speaks text parts and skips think parts', () => {+ const content: TMessageContentParts[] = [+ { type: ContentTypes.THINK, think: 'internal reasoning' },+ { type: ContentTypes.TEXT, text: 'First.' },+ { type: ContentTypes.THINK, think: 'more reasoning' },+ { type: ContentTypes.TEXT, text: 'Second.' },+ ];+ expect(getSpeechText({ content, text: 'ignored' })).toBe('First. Second.');+ });++ test('is empty for a response that only reasoned', () => {+ const content: TMessageContentParts[] = [{ type: ContentTypes.THINK, think: 'reasoning' }];+ expect(getSpeechText({ content })).toBe('');+ });++ test('never speaks steer parts', () => {+ const content: TMessageContentParts[] = [+ { type: ContentTypes.STEER, steer: 'user words' } as TMessageContentParts,+ { type: ContentTypes.TEXT, text: 'answer' },+ ];+ expect(getSpeechText({ content })).toBe('answer');+ });++ test('falls back to plain text when there are no content parts', () => {+ expect(getSpeechText({ content: [], text: 'plain answer' })).toBe('plain answer');+ expect(getSpeechText({ text: 'plain answer' })).toBe('plain answer');+ expect(getSpeechText({ content: null, text: null })).toBe('');+ });++ test('speaks string content, as imported messages may carry it', () => {+ const content = ':::thinking\nWork it out.\n:::\nThe answer is 4.';+ expect(getSpeechText({ content, text: '' })).toBe('The answer is 4.');+ });++ test('falls back to text when string content is empty', () => {+ expect(getSpeechText({ content: '', text: 'stored answer' })).toBe('stored answer');+ });++ test('strips a legacy thinking block from plain text', () => {+ const text = ':::thinking\nWork it out.\n:::\nThe answer is 4.';+ expect(getSpeechText({ text })).toBe('The answer is 4.');+ });++ test('speaks a repeated thinking block, which the message view shows as answer text', () => {+ const text = ':::thinking\na\n:::\nAnswer 1\n:::thinking\nb\n:::\nAnswer 2';+ expect(getSpeechText({ text })).toBe(parseThinkingContent(text).regularContent);+ expect(getSpeechText({ text })).toBe('Answer 1\n:::thinking\nb\n:::\nAnswer 2');+ });++ test('trims structured answers so whitespace alone is not speakable', () => {+ const content: TMessageContentParts[] = [+ { type: ContentTypes.THINK, think: 'reasoning' },+ { type: ContentTypes.TEXT, text: '\n' },+ ];+ expect(getSpeechText({ content })).toBe('');+ });++ test('speaks an unmatched marker, which the UI shows as answer text', () => {+ const text = 'Wrap reasoning in a :::thinking block.';+ expect(getSpeechText({ text })).toBe(text);+ });++ test('keeps plain text that merely uses colons', () => {+ expect(getSpeechText({ text: 'Ratio is 3:1 ::: done' })).toBe('Ratio is 3:1 ::: done');+ });+});+describe('parseTextParts', () => {test('should concatenate text parts', () => {const parts: TMessageContentParts[] = [diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.tsindex bf9ad180e7..a423fe455d 100644--- a/packages/data-provider/src/parsers.ts+++ b/packages/data-provider/src/parsers.ts@@ -416,6 +416,40 @@ export function parseTextParts(return result;}+const thinkingBlockPattern = /:::thinking([\s\S]*?):::/;++/**+ * Splits legacy plain-text reasoning from the answer exactly as the message view renders it:+ * the first `:::thinking ... :::` block becomes the collapsible Thoughts, the rest is the answer.+ */+export function parseThinkingContent(text: string): {+ thinkingContent: string;+ regularContent: string;+} {+ const thinkingMatch = text.match(thinkingBlockPattern);+ return {+ thinkingContent: thinkingMatch ? thinkingMatch[1].trim() : '',+ regularContent: thinkingMatch ? text.replace(thinkingBlockPattern, '').trim() : text,+ };+}++/**+ * The text Read Aloud speaks for a message: the answer the message view shows, never its+ * reasoning. Covers both reasoning shapes the UI collapses into Thoughts: `think` content parts+ * and the legacy `:::thinking ... :::` block in plain text.+ */+export function getSpeechText(message: {+ content?: Array<a.TMessageContentParts | undefined> | string | null;+ text?: string | null;+}): string {+ if (Array.isArray(message.content) && message.content.length > 0) {+ return parseTextParts(message.content, true).trim();+ }+ const text =+ typeof message.content === 'string' && message.content ? message.content : message.text;+ return parseThinkingContent(text ?? '').regularContent.trim();+}+export const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n', '```'];export function findLastSeparatorIndex(text: string, separators = SEPARATORS): number {