Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🗣️ fix(tts): Add Text Parser for Message Content Parts (#2840)
Danny Avila8e6668357730
diff --git a/api/server/services/Files/Audio/streamAudio-wip.js b/api/server/services/Files/Audio/streamAudio.spec.jssimilarity index 85%rename from api/server/services/Files/Audio/streamAudio-wip.jsrename to api/server/services/Files/Audio/streamAudio.spec.jsindex f472309496..6aee27c7b8 100644--- a/api/server/services/Files/Audio/streamAudio-wip.js+++ b/api/server/services/Files/Audio/streamAudio.spec.js@@ -13,7 +13,7 @@ describe('processChunks', () => {let processChunks;beforeEach(() => {- processChunks = createChunkProcessor();+ processChunks = createChunkProcessor('message-id');Message.findOne.mockClear();Message.findOne().lean.mockClear();});@@ -21,20 +21,17 @@ describe('processChunks', () => {it('should return an empty array when the message is not found', async () => {Message.findOne().lean.mockResolvedValueOnce(null);- const result = await processChunks('non-existent-id');+ const result = await processChunks();expect(result).toEqual([]);- expect(Message.findOne).toHaveBeenCalledWith(- { messageId: 'non-existent-id' },- 'text unfinished',- );+ expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');expect(Message.findOne().lean).toHaveBeenCalled();});it('should return an empty array when the message does not have a text property', async () => {Message.findOne().lean.mockResolvedValueOnce({ unfinished: true });- const result = await processChunks('message-id');+ const result = await processChunks();expect(result).toEqual([]);expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');@@ -45,7 +42,7 @@ describe('processChunks', () => {const messageText = 'This is a long message. It should be split into chunks. Lol hi mom';Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: true });- const result = await processChunks('message-id');+ const result = await processChunks();expect(result).toEqual([{ text: 'This is a long message. It should be split into chunks.', isFinished: false },@@ -58,7 +55,7 @@ describe('processChunks', () => {const messageText = 'This is a long message without separators hello there my friend';Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: true });- const result = await processChunks('message-id');+ const result = await processChunks();expect(result).toEqual([{ text: messageText, isFinished: false }]);expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');@@ -69,7 +66,7 @@ describe('processChunks', () => {const messageText = 'This is a finished message.';Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: false });- const result = await processChunks('message-id');+ const result = await processChunks();expect(result).toEqual([{ text: messageText, isFinished: true }]);expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');@@ -80,9 +77,9 @@ describe('processChunks', () => {const messageText = 'This is a finished message.';Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: false });- await processChunks('message-id');+ await processChunks();Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: false });- const result = await processChunks('message-id');+ const result = await processChunks();expect(result).toEqual([]);expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');diff --git a/client/src/components/Chat/Messages/HoverButtons.tsx b/client/src/components/Chat/Messages/HoverButtons.tsxindex 0a7c3e1d8d..ab9052dc77 100644--- a/client/src/components/Chat/Messages/HoverButtons.tsx+++ b/client/src/components/Chat/Messages/HoverButtons.tsx@@ -50,7 +50,7 @@ export default function HoverButtons({const [TextToSpeech] = useRecoilState<boolean>(store.TextToSpeech);const { handleMouseDown, handleMouseUp, toggleSpeech, isSpeaking, isLoading } = useTextToSpeech(- message?.text ?? '',+ message?.content ?? message?.text ?? '',isLast,index,);diff --git a/client/src/hooks/Input/useTextToSpeech.ts b/client/src/hooks/Input/useTextToSpeech.tsindex 0c1680ee3f..40378d24ba 100644--- a/client/src/hooks/Input/useTextToSpeech.ts+++ b/client/src/hooks/Input/useTextToSpeech.ts@@ -1,11 +1,13 @@import { useRef } from 'react';-import useTextToSpeechBrowser from './useTextToSpeechBrowser';+import { parseTextParts } from 'librechat-data-provider';+import type { TMessageContentParts } from 'librechat-data-provider';import useTextToSpeechExternal from './useTextToSpeechExternal';+import useTextToSpeechBrowser from './useTextToSpeechBrowser';import { usePauseGlobalAudio } from '../Audio';import { useRecoilState } from 'recoil';import store from '~/store';-const useTextToSpeech = (message: string, isLast: boolean, index = 0) => {+const useTextToSpeech = (message: string | TMessageContentParts[], isLast: boolean, index = 0) => {const [endpointTTS] = useRecoilState<string>(store.endpointTTS);const useExternalTextToSpeech = endpointTTS === 'external';@@ -34,7 +36,8 @@ const useTextToSpeech = (message: string, isLast: boolean, index = 0) => {isMouseDownRef.current = true;timerRef.current = window.setTimeout(() => {if (isMouseDownRef.current) {- generateSpeech(message, true);+ const parsedMessage = typeof message === 'string' ? message : parseTextParts(message);+ generateSpeech(parsedMessage, true);}}, 1000);};@@ -51,7 +54,8 @@ const useTextToSpeech = (message: string, isLast: boolean, index = 0) => {cancelSpeech();pauseGlobalAudio();} else {- generateSpeech(message, false);+ const parsedMessage = typeof message === 'string' ? message : parseTextParts(message);+ generateSpeech(parsedMessage, false);}};diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.tsindex 5bf27cc1dc..3406a33f65 100644--- a/packages/data-provider/src/parsers.ts+++ b/packages/data-provider/src/parsers.ts@@ -1,6 +1,8 @@import type { ZodIssue } from 'zod';-import type { TConversation, TPreset } from './schemas';-import type { TConfig, TEndpointOption, TEndpointsConfig } from './types';+import type * as a from './types/assistants';+import type * as s from './schemas';+import type * as t from './types';+import { ContentTypes } from './types/assistants';import {EModelEndpoint,openAISchema,@@ -71,7 +73,7 @@ export function getEnabledEndpoints() {}/** Orders an existing EndpointsConfig object based on enabled endpoint/custom ordering */-export function orderEndpointsConfig(endpointsConfig: TEndpointsConfig) {+export function orderEndpointsConfig(endpointsConfig: t.TEndpointsConfig) {if (!endpointsConfig) {return {};}@@ -79,7 +81,7 @@ export function orderEndpointsConfig(endpointsConfig: TEndpointsConfig) {const endpointKeys = Object.keys(endpointsConfig);const defaultCustomIndex = enabledEndpoints.indexOf(EModelEndpoint.custom);return endpointKeys.reduce(- (accumulatedConfig: Record<string, TConfig | null | undefined>, currentEndpointKey) => {+ (accumulatedConfig: Record<string, t.TConfig | null | undefined>, currentEndpointKey) => {const isCustom = !(currentEndpointKey in EModelEndpoint);const isEnabled = enabledEndpoints.includes(currentEndpointKey);if (!isEnabled && !isCustom) {@@ -91,7 +93,7 @@ export function orderEndpointsConfig(endpointsConfig: TEndpointsConfig) {if (isCustom) {accumulatedConfig[currentEndpointKey] = {order: defaultCustomIndex >= 0 ? defaultCustomIndex : 9999,- ...(endpointsConfig[currentEndpointKey] as Omit<TConfig, 'order'> & { order?: number }),+ ...(endpointsConfig[currentEndpointKey] as Omit<t.TConfig, 'order'> & { order?: number }),};} else if (endpointsConfig[currentEndpointKey]) {accumulatedConfig[currentEndpointKey] = {@@ -165,7 +167,7 @@ export const parseConvo = ({}: {endpoint: EModelEndpoint;endpointType?: EModelEndpoint;- conversation: Partial<TConversation | TPreset>;+ conversation: Partial<s.TConversation | s.TPreset>;possibleValues?: TPossibleValues;// TODO: POC for default schema// defaultSchema?: Partial<EndpointSchema>,@@ -182,7 +184,7 @@ export const parseConvo = ({// schema = schemaCreators[endpoint](defaultSchema);// }- const convo = schema.parse(conversation) as TConversation;+ const convo = schema.parse(conversation) as s.TConversation;const { models, secondaryModels } = possibleValues ?? {};if (models && convo) {@@ -196,7 +198,7 @@ export const parseConvo = ({return convo;};-export const getResponseSender = (endpointOption: TEndpointOption): string => {+export const getResponseSender = (endpointOption: t.TEndpointOption): string => {const { model, endpoint, endpointType, modelDisplayLabel, chatGptLabel, modelLabel, jailbreak } =endpointOption;@@ -292,7 +294,7 @@ export const parseCompactConvo = ({}: {endpoint?: EModelEndpoint;endpointType?: EModelEndpoint;- conversation: Partial<TConversation | TPreset>;+ conversation: Partial<s.TConversation | s.TPreset>;possibleValues?: TPossibleValues;// TODO: POC for default schema// defaultSchema?: Partial<EndpointSchema>,@@ -309,7 +311,7 @@ export const parseCompactConvo = ({schema = compactEndpointSchemas[endpointType];}- const convo = schema.parse(conversation) as TConversation;+ const convo = schema.parse(conversation) as s.TConversation;// const { models, secondaryModels } = possibleValues ?? {};const { models } = possibleValues ?? {};@@ -323,3 +325,25 @@ export const parseCompactConvo = ({return convo;};++export function parseTextParts(contentParts: a.TMessageContentParts[]): string {+ let result = '';++ for (const part of contentParts) {+ if (part.type === ContentTypes.TEXT) {+ const textValue = part.text.value;++ if (+ result.length > 0 &&+ textValue.length > 0 &&+ result[result.length - 1] !== ' ' &&+ textValue[0] !== ' '+ ) {+ result += ' ';+ }+ result += textValue;+ }+ }++ return result;+}