Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
⏯️ fix(tts): Resolve Voice Selection and Manual Playback Issues (#2845)
Danny Avila514a502b9c3a
diff --git a/api/server/middleware/checkBan.js b/api/server/middleware/checkBan.jsindex aa322cd1c2..ee7d7171de 100644--- a/api/server/middleware/checkBan.js+++ b/api/server/middleware/checkBan.js@@ -2,14 +2,12 @@ const Keyv = require('keyv');const uap = require('ua-parser-js');const { ViolationTypes } = require('librechat-data-provider');const { isEnabled, removePorts } = require('../utils');-const keyvRedis = require('~/cache/keyvRedis');+const keyvMongo = require('~/cache/keyvMongo');const denyRequest = require('./denyRequest');const { getLogStores } = require('~/cache');const User = require('~/models/User');-const banCache = isEnabled(process.env.USE_REDIS)- ? new Keyv({ store: keyvRedis })- : new Keyv({ namespace: ViolationTypes.BAN, ttl: 0 });+const banCache = new Keyv({ store: keyvMongo, namespace: ViolationTypes.BAN, ttl: 0 });const message = 'Your account has been temporarily banned due to violations of our service.';/**diff --git a/api/server/services/Files/Audio/streamAudio.js b/api/server/services/Files/Audio/streamAudio.jsindex 4b74816d28..9f301e710b 100644--- a/api/server/services/Files/Audio/streamAudio.js+++ b/api/server/services/Files/Audio/streamAudio.js@@ -90,7 +90,7 @@ function findLastSeparatorIndex(text, separators = SEPARATORS) {}const MAX_NOT_FOUND_COUNT = 6;-const MAX_NO_CHANGE_COUNT = 12;+const MAX_NO_CHANGE_COUNT = 10;/*** @param {string} messageId@@ -152,6 +152,64 @@ function createChunkProcessor(messageId) {return processChunks;}+/**+ * @param {string} text+ * @param {number} [chunkSize=4000]+ * @returns {{ text: string, isFinished: boolean }[]}+ */+function splitTextIntoChunks(text, chunkSize = 4000) {+ if (!text) {+ throw new Error('Text is required');+ }++ const chunks = [];+ let startIndex = 0;+ const textLength = text.length;++ while (startIndex < textLength) {+ let endIndex = Math.min(startIndex + chunkSize, textLength);+ let chunkText = text.slice(startIndex, endIndex);++ if (endIndex < textLength) {+ let lastSeparatorIndex = -1;+ for (const separator of SEPARATORS) {+ const index = chunkText.lastIndexOf(separator);+ if (index !== -1) {+ lastSeparatorIndex = Math.max(lastSeparatorIndex, index);+ }+ }++ if (lastSeparatorIndex !== -1) {+ endIndex = startIndex + lastSeparatorIndex + 1;+ chunkText = text.slice(startIndex, endIndex);+ } else {+ const nextSeparatorIndex = text.slice(endIndex).search(/\S/);+ if (nextSeparatorIndex !== -1) {+ endIndex += nextSeparatorIndex;+ chunkText = text.slice(startIndex, endIndex);+ }+ }+ }++ chunkText = chunkText.trim();+ if (chunkText) {+ chunks.push({+ text: chunkText,+ isFinished: endIndex >= textLength,+ });+ } else if (chunks.length > 0) {+ chunks[chunks.length - 1].isFinished = true;+ }++ startIndex = endIndex;+ while (startIndex < textLength && text[startIndex].trim() === '') {+ startIndex++;+ }+ }++ return chunks;+}+/*** Input stream text to speech* @param {Express.Response} res@@ -307,6 +365,7 @@ module.exports = {inputStreamTextToSpeech,findLastSeparatorIndex,createChunkProcessor,+ splitTextIntoChunks,llmMessageSource,getRandomVoiceId,};diff --git a/api/server/services/Files/Audio/streamAudio.spec.js b/api/server/services/Files/Audio/streamAudio.spec.jsindex 6aee27c7b8..7aff8dbfa7 100644--- a/api/server/services/Files/Audio/streamAudio.spec.js+++ b/api/server/services/Files/Audio/streamAudio.spec.js@@ -1,5 +1,5 @@+const { createChunkProcessor, splitTextIntoChunks } = require('./streamAudio');const { Message } = require('~/models/Message');-const { createChunkProcessor } = require('./streamAudio');jest.mock('~/models/Message', () => ({Message: {@@ -86,3 +86,52 @@ describe('processChunks', () => {expect(Message.findOne().lean).toHaveBeenCalledTimes(2);});});++describe('splitTextIntoChunks', () => {+ test('splits text into chunks of specified size with default separators', () => {+ const text = 'This is a test. This is only a test! Make sure it works properly? Okay.';+ const chunkSize = 20;+ const expectedChunks = [+ { text: 'This is a test.', isFinished: false },+ { text: 'This is only a test!', isFinished: false },+ { text: 'Make sure it works p', isFinished: false },+ { text: 'roperly? Okay.', isFinished: true },+ ];++ const result = splitTextIntoChunks(text, chunkSize);+ expect(result).toEqual(expectedChunks);+ });++ test('splits text into chunks with default size', () => {+ const text = 'A'.repeat(8000) + '. The end.';+ const expectedChunks = [+ { text: 'A'.repeat(4000), isFinished: false },+ { text: 'A'.repeat(4000), isFinished: false },+ { text: '. The end.', isFinished: true },+ ];++ const result = splitTextIntoChunks(text);+ expect(result).toEqual(expectedChunks);+ });++ test('returns a single chunk if text length is less than chunk size', () => {+ const text = 'Short text.';+ const expectedChunks = [{ text: 'Short text.', isFinished: true }];++ const result = splitTextIntoChunks(text, 4000);+ expect(result).toEqual(expectedChunks);+ });++ test('handles text with no separators correctly', () => {+ const text = 'ThisTextHasNoSeparatorsAndIsVeryLong'.repeat(100);+ const chunkSize = 4000;+ const expectedChunks = [{ text: text, isFinished: true }];++ const result = splitTextIntoChunks(text, chunkSize);+ expect(result).toEqual(expectedChunks);+ });++ test('throws an error when text is empty', () => {+ expect(() => splitTextIntoChunks('')).toThrow('Text is required');+ });+});diff --git a/api/server/services/Files/Audio/textToSpeech.js b/api/server/services/Files/Audio/textToSpeech.jsindex 6c8f306c89..2d77324ce4 100644--- a/api/server/services/Files/Audio/textToSpeech.js+++ b/api/server/services/Files/Audio/textToSpeech.js@@ -1,6 +1,6 @@const axios = require('axios');const getCustomConfig = require('~/server/services/Config/getCustomConfig');-const { getRandomVoiceId, createChunkProcessor } = require('./streamAudio');+const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio');const { extractEnvVariable } = require('librechat-data-provider');const { logger } = require('~/config');@@ -54,7 +54,7 @@ function removeUndefined(obj) {* This function prepares the necessary data and headers for making a request to the OpenAI TTS* It uses the provided TTS schema, input text, and voice to create the request*- * @param {Object} ttsSchema - The TTS schema containing the OpenAI configuration+ * @param {TCustomConfig['tts']['openai']} ttsSchema - The TTS schema containing the OpenAI configuration* @param {string} input - The text to be converted to speech* @param {string} voice - The voice to be used for the speech*@@ -62,27 +62,27 @@ function removeUndefined(obj) {* If an error occurs, it throws an error with a message indicating that the selected voice is not available*/function openAIProvider(ttsSchema, input, voice) {- const url = ttsSchema.openai?.url || 'https://api.openai.com/v1/audio/speech';+ const url = ttsSchema?.url || 'https://api.openai.com/v1/audio/speech';if (- ttsSchema.openai?.voices &&- ttsSchema.openai.voices.length > 0 &&- !ttsSchema.openai.voices.includes(voice) &&- !ttsSchema.openai.voices.includes('ALL')+ ttsSchema?.voices &&+ ttsSchema.voices.length > 0 &&+ !ttsSchema.voices.includes(voice) &&+ !ttsSchema.voices.includes('ALL')) {throw new Error(`Voice ${voice} is not available.`);}let data = {input,- model: ttsSchema.openai?.model,- voice: ttsSchema.openai?.voices && ttsSchema.openai.voices.length > 0 ? voice : undefined,- backend: ttsSchema.openai?.backend,+ model: ttsSchema?.model,+ voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,+ backend: ttsSchema?.backend,};let headers = {'Content-Type': 'application/json',- Authorization: 'Bearer ' + extractEnvVariable(ttsSchema.openai?.apiKey),+ Authorization: 'Bearer ' + extractEnvVariable(ttsSchema?.apiKey),};[data, headers].forEach(removeUndefined);@@ -95,7 +95,7 @@ function openAIProvider(ttsSchema, input, voice) {* This function prepares the necessary data and headers for making a request to the Eleven Labs TTS* It uses the provided TTS schema, input text, and voice to create the request*- * @param {Object} ttsSchema - The TTS schema containing the Eleven Labs configuration+ * @param {TCustomConfig['tts']['elevenLabs']} ttsSchema - The TTS schema containing the Eleven Labs configuration* @param {string} input - The text to be converted to speech* @param {string} voice - The voice to be used for the speech* @param {boolean} stream - Whether to stream the audio or not@@ -105,34 +105,31 @@ function openAIProvider(ttsSchema, input, voice) {*/function elevenLabsProvider(ttsSchema, input, voice, stream) {let url =- ttsSchema.elevenlabs?.url ||+ ttsSchema?.url ||`https://api.elevenlabs.io/v1/text-to-speech/{voice_id}${stream ? '/stream' : ''}`;- if (- !ttsSchema.elevenlabs?.voices.includes(voice) &&- !ttsSchema.elevenlabs?.voices.includes('ALL')- ) {+ if (!ttsSchema?.voices.includes(voice) && !ttsSchema?.voices.includes('ALL')) {throw new Error(`Voice ${voice} is not available.`);}url = url.replace('{voice_id}', voice);let data = {- model_id: ttsSchema.elevenlabs?.model,+ model_id: ttsSchema?.model,text: input,// voice_id: voice,voice_settings: {- similarity_boost: ttsSchema.elevenlabs?.voice_settings?.similarity_boost,- stability: ttsSchema.elevenlabs?.voice_settings?.stability,- style: ttsSchema.elevenlabs?.voice_settings?.style,- use_speaker_boost: ttsSchema.elevenlabs?.voice_settings?.use_speaker_boost || undefined,+ similarity_boost: ttsSchema?.voice_settings?.similarity_boost,+ stability: ttsSchema?.voice_settings?.stability,+ style: ttsSchema?.voice_settings?.style,+ use_speaker_boost: ttsSchema?.voice_settings?.use_speaker_boost || undefined,},- pronunciation_dictionary_locators: ttsSchema.elevenlabs?.pronunciation_dictionary_locators,+ pronunciation_dictionary_locators: ttsSchema?.pronunciation_dictionary_locators,};let headers = {'Content-Type': 'application/json',- 'xi-api-key': extractEnvVariable(ttsSchema.elevenlabs?.apiKey),+ 'xi-api-key': extractEnvVariable(ttsSchema?.apiKey),Accept: 'audio/mpeg',};@@ -146,7 +143,7 @@ function elevenLabsProvider(ttsSchema, input, voice, stream) {* This function prepares the necessary data and headers for making a request to the LocalAI TTS* It uses the provided TTS schema, input text, and voice to create the request*- * @param {Object} ttsSchema - The TTS schema containing the LocalAI configuration+ * @param {TCustomConfig['tts']['localai']} ttsSchema - The TTS schema containing the LocalAI configuration* @param {string} input - The text to be converted to speech* @param {string} voice - The voice to be used for the speech*@@ -154,102 +151,78 @@ function elevenLabsProvider(ttsSchema, input, voice, stream) {* @throws {Error} Throws an error if the selected voice is not available*/function localAIProvider(ttsSchema, input, voice) {- let url = ttsSchema.localai?.url;+ let url = ttsSchema?.url;if (- ttsSchema.localai?.voices &&- ttsSchema.localai.voices.length > 0 &&- !ttsSchema.localai.voices.includes(voice) &&- !ttsSchema.localai.voices.includes('ALL')+ ttsSchema?.voices &&+ ttsSchema.voices.length > 0 &&+ !ttsSchema.voices.includes(voice) &&+ !ttsSchema.voices.includes('ALL')) {throw new Error(`Voice ${voice} is not available.`);}let data = {input,- model: ttsSchema.localai?.voices && ttsSchema.localai.voices.length > 0 ? voice : undefined,- backend: ttsSchema.localai?.backend,+ model: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,+ backend: ttsSchema?.backend,};let headers = {'Content-Type': 'application/json',- Authorization: 'Bearer ' + extractEnvVariable(ttsSchema.localai?.apiKey),+ Authorization: 'Bearer ' + extractEnvVariable(ttsSchema?.apiKey),};[data, headers].forEach(removeUndefined);- if (extractEnvVariable(ttsSchema.localai.apiKey) === '') {+ if (extractEnvVariable(ttsSchema.apiKey) === '') {delete headers.Authorization;}return [url, data, headers];}-/* not used */-/*-async function streamAudioFromWebSocket(req, res) {- const { voice } = req.body;- const customConfig = await getCustomConfig();-- if (!customConfig) {- return res.status(500).send('Custom config not found');- }-- const ttsSchema = customConfig.tts;- const provider = getProvider(ttsSchema);-- if (provider !== 'elevenlabs') {- return res.status(400).send('WebSocket streaming is only supported for Eleven Labs');- }-- const url =- ttsSchema.elevenlabs.websocketUrl ||- 'wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id={model}'- .replace('{voice_id}', voice)- .replace('{model}', ttsSchema.elevenlabs.model);- const ws = new WebSocket(url);-- ws.onopen = () => {- logger.debug('WebSocket connection opened');- sendTextToWebsocket(ws, (data) => {- res.write(data); // Stream data directly to the response- });- };-- ws.onclose = () => {- logger.debug('WebSocket connection closed');- res.end(); // End the response when the WebSocket is closed- };-- ws.onerror = (error) => {- logger.error('WebSocket error:', error);- res.status(500).send('WebSocket error');- };+/**+ *+ * Returns provider and its schema for use with TTS requests+ * @param {TCustomConfig} customConfig+ * @param {string} _voice+ * @returns {Promise<[string, TProviderSchema]>}+ */+async function getProviderSchema(customConfig) {+ const provider = getProvider(customConfig.tts);+ return [provider, customConfig.tts[provider]];}-*//***- * @param {TCustomConfig} customConfig- * @param {string} voice- * @returns {Promise<ArrayBuffer>}+ * Returns a tuple of the TTS schema as well as the voice for the TTS request+ * @param {TProviderSchema} providerSchema+ * @param {string} requestVoice+ * @returns {Promise<string>}*/-async function ttsRequest(- customConfig,- { input, voice: _v, stream = true } = { input: '', stream: true },-) {- const ttsSchema = customConfig.tts;- const provider = getProvider(ttsSchema);- const voices = ttsSchema[provider].voices.filter(- (voice) => voice && voice.toUpperCase() !== 'ALL',- );- let voice = _v;+async function getVoice(providerSchema, requestVoice) {+ const voices = providerSchema.voices.filter((voice) => voice && voice.toUpperCase() !== 'ALL');+ let voice = requestVoice;if (!voice || !voices.includes(voice) || (voice.toUpperCase() === 'ALL' && voices.length > 1)) {voice = getRandomVoiceId(voices);}- let [url, data, headers] = [];+ return voice;+}+/**+ *+ * @param {string} provider+ * @param {TProviderSchema} ttsSchema+ * @param {object} params+ * @param {string} params.voice+ * @param {string} params.input+ * @param {boolean} [params.stream]+ * @returns {Promise<ArrayBuffer>}+ */+async function ttsRequest(provider, ttsSchema, { input, voice, stream = true } = { stream: true }) {+ let [url, data, headers] = [];switch (provider) {case 'openai':[url, data, headers] = openAIProvider(ttsSchema, input, voice);@@ -283,7 +256,7 @@ async function ttsRequest(* @throws {Error} Throws an error if the provider is invalid*/async function textToSpeech(req, res) {- const { input, voice } = req.body;+ const { input } = req.body;if (!input) {return res.status(400).send('Missing text in request body');@@ -296,8 +269,47 @@ async function textToSpeech(req, res) {try {res.setHeader('Content-Type', 'audio/mpeg');- const response = await ttsRequest(customConfig, { input, voice });- response.data.pipe(res);+ const [provider, ttsSchema] = await getProviderSchema(customConfig);+ const voice = await getVoice(ttsSchema, req.body.voice);+ if (input.length < 4096) {+ const response = await ttsRequest(provider, ttsSchema, { input, voice });+ response.data.pipe(res);+ return;+ }++ const textChunks = splitTextIntoChunks(input, 1000);++ for (const chunk of textChunks) {+ try {+ const response = await ttsRequest(provider, ttsSchema, {+ voice,+ input: chunk.text,+ stream: true,+ });++ logger.debug(`[textToSpeech] user: ${req?.user?.id} | writing audio stream`);+ await new Promise((resolve) => {+ response.data.pipe(res, { end: chunk.isFinished });+ response.data.on('end', () => {+ resolve();+ });+ });++ if (chunk.isFinished) {+ break;+ }+ } catch (innerError) {+ logger.error('Error processing update:', chunk, innerError);+ if (!res.headersSent) {+ res.status(500).end();+ }+ return;+ }+ }++ if (!res.headersSent) {+ res.end();+ }} catch (error) {logger.error('An error occurred while creating the audio stream:', error);res.status(500).send('An error occurred');@@ -311,8 +323,17 @@ async function streamAudio(req, res) {return res.status(500).send('Custom config not found');}+ const [provider, ttsSchema] = await getProviderSchema(customConfig);+ const voice = await getVoice(ttsSchema, req.body.voice);+try {let shouldContinue = true;++ req.on('close', () => {+ logger.warn('[streamAudio] Audio Stream Request closed by client');+ shouldContinue = false;+ });+const processChunks = createChunkProcessor(req.body.messageId);while (shouldContinue) {@@ -337,7 +358,8 @@ async function streamAudio(req, res) {for (const update of updates) {try {- const response = await ttsRequest(customConfig, {+ const response = await ttsRequest(provider, ttsSchema, {+ voice,input: update.text,stream: true,});@@ -348,7 +370,7 @@ async function streamAudio(req, res) {logger.debug(`[streamAudio] user: ${req?.user?.id} | writing audio stream`);await new Promise((resolve) => {- response.data.pipe(res, { end: false });+ response.data.pipe(res, { end: update.isFinished });response.data.on('end', () => {resolve();});diff --git a/api/server/services/start/assistants.js b/api/server/services/start/assistants.jsindex eba6319f84..aa43f2b7c8 100644--- a/api/server/services/start/assistants.js+++ b/api/server/services/start/assistants.js@@ -1,6 +1,5 @@const {Capabilities,- EModelEndpoint,assistantEndpointSchema,defaultAssistantsVersion,} = require('librechat-data-provider');diff --git a/api/typedefs.js b/api/typedefs.jsindex 5c83cab159..b55dfa120a 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -349,6 +349,12 @@* @memberof typedefs*/+/**+ * @exports TProviderSchema+ * @typedef {import('librechat-data-provider').TProviderSchema} TProviderSchema+ * @memberof typedefs+ */+/*** @exports TEndpoint* @typedef {import('librechat-data-provider').TEndpoint} TEndpointdiff --git a/client/src/components/Chat/Input/StreamAudio.tsx b/client/src/components/Chat/Input/StreamAudio.tsxindex 34bd4a3039..b89c295d37 100644--- a/client/src/components/Chat/Input/StreamAudio.tsx+++ b/client/src/components/Chat/Input/StreamAudio.tsx@@ -1,10 +1,10 @@import { useParams } from 'react-router-dom';+import { useEffect, useCallback } from 'react';import { QueryKeys } from 'librechat-data-provider';import { useQueryClient } from '@tanstack/react-query';-import { useEffect, useCallback } from 'react';import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';import type { TMessage } from 'librechat-data-provider';-import { useCustomAudioRef, MediaSourceAppender } from '~/hooks/Audio';+import { useCustomAudioRef, MediaSourceAppender, usePauseGlobalAudio } from '~/hooks/Audio';import { useAuthContext } from '~/hooks';import { globalAudioId } from '~/common';import store from '~/store';@@ -24,6 +24,7 @@ export default function StreamAudio({ index = 0 }) {const cacheTTS = useRecoilValue(store.cacheTTS);const playbackRate = useRecoilValue(store.playbackRate);+ const voice = useRecoilValue(store.voice);const activeRunId = useRecoilValue(store.activeRunFamily(index));const automaticPlayback = useRecoilValue(store.automaticPlayback);const isSubmitting = useRecoilValue(store.isSubmittingFamily(index));@@ -34,6 +35,7 @@ export default function StreamAudio({ index = 0 }) {const [globalAudioURL, setGlobalAudioURL] = useRecoilState(store.globalAudioURLFamily(index));const { audioRef } = useCustomAudioRef({ setIsPlaying });+ const { pauseGlobalAudio } = usePauseGlobalAudio();const { conversationId: paramId } = useParams();const queryParam = paramId === 'new' ? paramId : latestMessage?.conversationId ?? paramId ?? '';@@ -90,7 +92,7 @@ export default function StreamAudio({ index = 0 }) {const response = await fetch('/api/files/tts', {method: 'POST',headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` },- body: JSON.stringify({ messageId: latestMessage?.messageId, runId: activeRunId }),+ body: JSON.stringify({ messageId: latestMessage?.messageId, runId: activeRunId, voice }),});if (!response.ok) {@@ -166,6 +168,7 @@ export default function StreamAudio({ index = 0 }) {audioRunId,cacheTTS,audioRef,+ voice,token,]);@@ -180,6 +183,12 @@ export default function StreamAudio({ index = 0 }) {}}, [audioRef, globalAudioURL, playbackRate]);+ useEffect(() => {+ pauseGlobalAudio();+ // We only want the effect to run when the paramId changes+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [paramId]);+return (<audioref={audioRef}diff --git a/client/src/hooks/Input/useTextToSpeechExternal.ts b/client/src/hooks/Input/useTextToSpeechExternal.tsindex 709b4606d7..58dc82e6c4 100644--- a/client/src/hooks/Input/useTextToSpeechExternal.ts+++ b/client/src/hooks/Input/useTextToSpeechExternal.ts@@ -1,6 +1,7 @@import { useRecoilValue } from 'recoil';-import { useCallback, useEffect, useState, useMemo } from 'react';+import { useCallback, useEffect, useState, useMemo, useRef } from 'react';import { useTextToSpeechMutation } from '~/data-provider';+import useLocalize from '~/hooks/useLocalize';import { useToastContext } from '~/Providers';import store from '~/store';@@ -12,16 +13,16 @@ const createFormData = (text: string, voice: string) => {};function useTextToSpeechExternal(isLast: boolean, index = 0) {+ const localize = useLocalize();const { showToast } = useToastContext();const voice = useRecoilValue(store.voice);const cacheTTS = useRecoilValue(store.cacheTTS);const playbackRate = useRecoilValue(store.playbackRate);- const [text, setText] = useState<string | null>(null);+ const audioRef = useRef<HTMLAudioElement | null>(null);+const [downloadFile, setDownloadFile] = useState(false);const [isLocalSpeaking, setIsSpeaking] = useState(false);- const [blobUrl, setBlobUrl] = useState<string | null>(null);- const [audio, setAudio] = useState<HTMLAudioElement | null>(null);/* Global Audio Variables */const globalIsFetching = useRecoilValue(store.globalAudioFetchingFamily(index));@@ -29,10 +30,13 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {const playAudio = (blobUrl: string) => {const newAudio = new Audio(blobUrl);- if (playbackRate && playbackRate !== 1) {- newAudio.playbackRate = playbackRate;- }+ const initializeAudio = () => {+ if (playbackRate && playbackRate !== 1) {+ newAudio.playbackRate = playbackRate;+ }+ };+ initializeAudio();const playPromise = () => newAudio.play().then(() => setIsSpeaking(true));playPromise().catch((error: Error) => {@@ -40,10 +44,12 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {error?.message &&error.message.includes('The play() request was interrupted by a call to pause()')) {+ console.log('Play request was interrupted by a call to pause()');+ initializeAudio();return playPromise().catch(console.error);}console.error(error);- showToast({ message: `Error playing audio: ${error.message}`, status: 'error' });+ showToast({ message: localize('com_nav_audio_play_error', error.message), status: 'error' });});newAudio.onended = () => {@@ -52,8 +58,7 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {setIsSpeaking(false);};- setAudio(newAudio);- setBlobUrl(blobUrl);+ audioRef.current = newAudio;};const downloadAudio = (blobUrl: string) => {@@ -65,35 +70,32 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {};const { mutate: processAudio, isLoading: isProcessing } = useTextToSpeechMutation({- onSuccess: async (data: ArrayBuffer) => {+ onMutate: (variables) => {+ const inputText = (variables.get('input') ?? '') as string;+ if (inputText.length >= 4096) {+ showToast({+ message: localize('com_nav_long_audio_warning'),+ status: 'warning',+ });+ }+ },+ onSuccess: async (data: ArrayBuffer, variables) => {try {- const mediaSource = new MediaSource();- const audio = new Audio();- audio.src = URL.createObjectURL(mediaSource);- audio.autoplay = true;-- mediaSource.onsourceopen = () => {- const sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');- sourceBuffer.appendBuffer(data);- };+ const inputText = (variables.get('input') ?? '') as string;+ const audioBlob = new Blob([data], { type: 'audio/mpeg' });- audio.onended = () => {- URL.revokeObjectURL(audio.src);- setIsSpeaking(false);- };-- setAudio(audio);-- if (cacheTTS) {+ if (cacheTTS && inputText) {const cache = await caches.open('tts-responses');- const request = new Request(text!);- const response = new Response(new Blob([data], { type: 'audio/mpeg' }));+ const request = new Request(inputText!);+ const response = new Response(audioBlob);cache.put(request, response);}+ const blobUrl = URL.createObjectURL(audioBlob);if (downloadFile) {- downloadAudio(audio.src);+ downloadAudio(blobUrl);}+ playAudio(blobUrl);} catch (error) {showToast({message: `Error processing audio: ${(error as Error).message}`,@@ -102,13 +104,15 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {}},onError: (error: unknown) => {- showToast({ message: `Error: ${(error as Error).message}`, status: 'error' });+ showToast({+ message: localize('com_nav_audio_process_error', (error as Error).message),+ status: 'error',+ });},});const generateSpeechExternal = async (text: string, download: boolean) => {- setText(text);- const cachedResponse = await getCachedResponse(text);+ const cachedResponse = await caches.match(text);if (cachedResponse && cacheTTS) {handleCachedResponse(cachedResponse, download);@@ -119,8 +123,6 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {}};- const getCachedResponse = async (text: string) => await caches.match(text);-const handleCachedResponse = async (cachedResponse: Response, download: boolean) => {const audioBlob = await cachedResponse.blob();const blobUrl = URL.createObjectURL(audioBlob);@@ -132,12 +134,13 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {};const cancelSpeech = useCallback(() => {- if (audio) {- audio.pause();- blobUrl && URL.revokeObjectURL(blobUrl);+ if (audioRef.current) {+ audioRef.current.pause();+ audioRef.current.src && URL.revokeObjectURL(audioRef.current.src);+ audioRef.current = null;setIsSpeaking(false);}- }, [audio, blobUrl]);+ }, []);useEffect(() => cancelSpeech, [cancelSpeech]);diff --git a/client/src/localization/languages/Eng.ts b/client/src/localization/languages/Eng.tsindex 7da4c622e2..4c22d681d1 100644--- a/client/src/localization/languages/Eng.ts+++ b/client/src/localization/languages/Eng.ts@@ -550,6 +550,9 @@ export default {com_nav_auto_transcribe_audio: 'Auto transcribe audio',com_nav_db_sensitivity: 'Decibel sensitivity',com_nav_playback_rate: 'Audio Playback Rate',+ com_nav_audio_play_error: 'Error playing audio: {0}',+ com_nav_audio_process_error: 'Error processing audio: {0}',+ com_nav_long_audio_warning: 'Longer texts will take longer to process.',com_nav_engine: 'Engine',com_nav_browser: 'Browser',com_nav_external: 'External',diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 3be5b2ee4c..e773a5376e 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -223,41 +223,41 @@ export const azureEndpointSchema = zexport type TAzureConfig = Omit<z.infer<typeof azureEndpointSchema>, 'groups'> &TAzureConfigValidationResult;-const ttsSchema = z.object({- openai: z- .object({- url: z.string().optional(),- apiKey: z.string(),- model: z.string(),- voices: z.array(z.string()),- })- .optional(),- elevenLabs: z- .object({- url: z.string().optional(),- websocketUrl: z.string().optional(),- apiKey: z.string(),- model: z.string(),- voices: z.array(z.string()),- voice_settings: z- .object({- similarity_boost: z.number().optional(),- stability: z.number().optional(),- style: z.number().optional(),- use_speaker_boost: z.boolean().optional(),- })- .optional(),- pronunciation_dictionary_locators: z.array(z.string()).optional(),- })- .optional(),- localai: z+const ttsOpenaiSchema = z.object({+ url: z.string().optional(),+ apiKey: z.string(),+ model: z.string(),+ voices: z.array(z.string()),+});++const ttsElevenLabsSchema = z.object({+ url: z.string().optional(),+ websocketUrl: z.string().optional(),+ apiKey: z.string(),+ model: z.string(),+ voices: z.array(z.string()),+ voice_settings: z.object({- url: z.string(),- apiKey: z.string().optional(),- voices: z.array(z.string()),- backend: z.string(),+ similarity_boost: z.number().optional(),+ stability: z.number().optional(),+ style: z.number().optional(),+ use_speaker_boost: z.boolean().optional(),}).optional(),+ pronunciation_dictionary_locators: z.array(z.string()).optional(),+});++const ttsLocalaiSchema = z.object({+ url: z.string(),+ apiKey: z.string().optional(),+ voices: z.array(z.string()),+ backend: z.string(),+});++const ttsSchema = z.object({+ openai: ttsOpenaiSchema.optional(),+ elevenLabs: ttsElevenLabsSchema.optional(),+ localai: ttsLocalaiSchema.optional(),});const sttSchema = z.object({@@ -359,6 +359,12 @@ export const getConfigDefaults = () => getSchemaDefaults(configSchema);export type TCustomConfig = z.infer<typeof configSchema>;+export type TProviderSchema =+ | z.infer<typeof ttsOpenaiSchema>+ | z.infer<typeof ttsElevenLabsSchema>+ | z.infer<typeof ttsLocalaiSchema>+ | undefined;+export enum KnownEndpoints {anyscale = 'anyscale',apipie = 'apipie',