Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔀 refactor: Modularize TTS Logic for Improved Browser support (#3657)

Danny Aviladba704079c44
diff --git a/client/src/common/types.ts b/client/src/common/types.ts
index e78cc5afe5..e79bf2bddf 100644
--- a/client/src/common/types.ts
+++ b/client/src/common/types.ts
@@ -19,6 +19,7 @@ import type {
TStartupConfig,
EModelEndpoint,
AssistantsEndpoint,
+ TMessageContentParts,
AuthorizationTypeEnum,
TSetOption as SetOption,
TokenExchangeMethodEnum,
@@ -31,6 +32,17 @@ export enum PromptsEditorMode {
ADVANCED = 'advanced',
}
+export enum STTEndpoints {
+ browser = 'browser',
+ external = 'external',
+}
+
+export enum TTSEndpoints {
+ browser = 'browser',
+ edge = 'edge',
+ external = 'external',
+}
+
export type AudioChunk = {
audio: string;
isFinal: boolean;
@@ -374,6 +386,19 @@ export type Option = Record<string, unknown> & {
value: string | number | null;
};
+export type VoiceOption = {
+ value: string;
+ label: string;
+};
+
+export type TMessageAudio = {
+ messageId?: string;
+ content?: TMessageContentParts[] | string;
+ className?: string;
+ isLast: boolean;
+ index: number;
+};
+
export type OptionWithIcon = Option & { icon?: React.ReactNode };
export type MentionOption = OptionWithIcon & {
type: string;
diff --git a/client/src/components/Audio/TTS.tsx b/client/src/components/Audio/TTS.tsx
new file mode 100644
index 0000000000..0ccad8a051
--- /dev/null
+++ b/client/src/components/Audio/TTS.tsx
@@ -0,0 +1,256 @@
+import { useEffect, useMemo } from 'react';
+import { useRecoilValue } from 'recoil';
+import type { TMessageAudio } from '~/common';
+import { useLocalize, useTTSBrowser, useTTSEdge, useTTSExternal } from '~/hooks';
+import { VolumeIcon, VolumeMuteIcon, Spinner } from '~/components/svg';
+import { useToastContext } from '~/Providers/ToastContext';
+import { logger } from '~/utils';
+import store from '~/store';
+
+export function BrowserTTS({ isLast, index, messageId, content, className }: TMessageAudio) {
+ const localize = useLocalize();
+ const playbackRate = useRecoilValue(store.playbackRate);
+
+ const { toggleSpeech, isSpeaking, isLoading, audioRef } = useTTSBrowser({
+ isLast,
+ index,
+ messageId,
+ content,
+ });
+
+ const renderIcon = (size: string) => {
+ if (isLoading === true) {
+ return <Spinner size={size} />;
+ }
+
+ if (isSpeaking === true) {
+ return <VolumeMuteIcon size={size} />;
+ }
+
+ return <VolumeIcon size={size} />;
+ };
+
+ useEffect(() => {
+ const messageAudio = document.getElementById(`audio-${messageId}`) as HTMLAudioElement | null;
+ if (!messageAudio) {
+ return;
+ }
+ if (playbackRate != null && playbackRate > 0 && messageAudio.playbackRate !== playbackRate) {
+ messageAudio.playbackRate = playbackRate;
+ }
+ }, [audioRef, isSpeaking, playbackRate, messageId]);
+
+ logger.log(
+ 'MessageAudio: audioRef.current?.src, audioRef.current',
+ audioRef.current?.src,
+ audioRef.current,
+ );
+
+ return (
+ <>
+ <button
+ className={className}
+ onClickCapture={() => {
+ if (audioRef.current) {
+ audioRef.current.muted = false;
+ }
+ toggleSpeech();
+ }}
+ type="button"
+ title={isSpeaking === true ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
+ >
+ {renderIcon('19')}
+ </button>
+ <audio
+ ref={audioRef}
+ controls
+ preload="none"
+ controlsList="nodownload nofullscreen noremoteplayback"
+ style={{
+ position: 'absolute',
+ overflow: 'hidden',
+ display: 'none',
+ height: '0px',
+ width: '0px',
+ }}
+ src={audioRef.current?.src}
+ onError={(error) => {
+ console.error('Error fetching audio:', error);
+ }}
+ id={`audio-${messageId}`}
+ muted
+ autoPlay
+ />
+ </>
+ );
+}
+
+export function EdgeTTS({ isLast, index, messageId, content, className }: TMessageAudio) {
+ const localize = useLocalize();
+ const playbackRate = useRecoilValue(store.playbackRate);
+ const isBrowserSupported = useMemo(
+ () => typeof MediaSource !== 'undefined' && MediaSource.isTypeSupported('audio/mpeg'),
+ [],
+ );
+
+ const { showToast } = useToastContext();
+ const { toggleSpeech, isSpeaking, isLoading, audioRef } = useTTSEdge({
+ isLast,
+ index,
+ messageId,
+ content,
+ });
+
+ const renderIcon = (size: string) => {
+ if (isLoading === true) {
+ return <Spinner size={size} />;
+ }
+
+ if (isSpeaking === true) {
+ return <VolumeMuteIcon size={size} />;
+ }
+
+ return <VolumeIcon size={size} />;
+ };
+
+ useEffect(() => {
+ const messageAudio = document.getElementById(`audio-${messageId}`) as HTMLAudioElement | null;
+ if (!messageAudio) {
+ return;
+ }
+ if (playbackRate != null && playbackRate > 0 && messageAudio.playbackRate !== playbackRate) {
+ messageAudio.playbackRate = playbackRate;
+ }
+ }, [audioRef, isSpeaking, playbackRate, messageId]);
+
+ logger.log(
+ 'MessageAudio: audioRef.current?.src, audioRef.current',
+ audioRef.current?.src,
+ audioRef.current,
+ );
+
+ return (
+ <>
+ <button
+ className={className}
+ onClickCapture={() => {
+ if (!isBrowserSupported) {
+ showToast({
+ message: localize('com_nav_tts_unsupported_error'),
+ status: 'error',
+ });
+ return;
+ }
+ if (audioRef.current) {
+ audioRef.current.muted = false;
+ }
+ toggleSpeech();
+ }}
+ type="button"
+ title={isSpeaking === true ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
+ >
+ {renderIcon('19')}
+ </button>
+ {isBrowserSupported ? (
+ <audio
+ ref={audioRef}
+ controls
+ preload="none"
+ controlsList="nodownload nofullscreen noremoteplayback"
+ style={{
+ position: 'absolute',
+ overflow: 'hidden',
+ display: 'none',
+ height: '0px',
+ width: '0px',
+ }}
+ src={audioRef.current?.src}
+ onError={(error) => {
+ console.error('Error fetching audio:', error);
+ }}
+ id={`audio-${messageId}`}
+ muted
+ autoPlay
+ />
+ ) : null}
+ </>
+ );
+}
+
+export function ExternalTTS({ isLast, index, messageId, content, className }: TMessageAudio) {
+ const localize = useLocalize();
+ const playbackRate = useRecoilValue(store.playbackRate);
+
+ const { toggleSpeech, isSpeaking, isLoading, audioRef } = useTTSExternal({
+ isLast,
+ index,
+ messageId,
+ content,
+ });
+
+ const renderIcon = (size: string) => {
+ if (isLoading === true) {
+ return <Spinner size={size} />;
+ }
+
+ if (isSpeaking === true) {
+ return <VolumeMuteIcon size={size} />;
+ }
+
+ return <VolumeIcon size={size} />;
+ };
+
+ useEffect(() => {
+ const messageAudio = document.getElementById(`audio-${messageId}`) as HTMLAudioElement | null;
+ if (!messageAudio) {
+ return;
+ }
+ if (playbackRate != null && playbackRate > 0 && messageAudio.playbackRate !== playbackRate) {
+ messageAudio.playbackRate = playbackRate;
+ }
+ }, [audioRef, isSpeaking, playbackRate, messageId]);
+
+ logger.log(
+ 'MessageAudio: audioRef.current?.src, audioRef.current',
+ audioRef.current?.src,
+ audioRef.current,
+ );
+
+ return (
+ <>
+ <button
+ className={className}
+ onClickCapture={() => {
+ if (audioRef.current) {
+ audioRef.current.muted = false;
+ }
+ toggleSpeech();
+ }}
+ type="button"
+ title={isSpeaking === true ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
+ >
+ {renderIcon('19')}
+ </button>
+ <audio
+ ref={audioRef}
+ controls
+ preload="none"
+ controlsList="nodownload nofullscreen noremoteplayback"
+ style={{
+ position: 'absolute',
+ overflow: 'hidden',
+ display: 'none',
+ height: '0px',
+ width: '0px',
+ }}
+ src={audioRef.current?.src}
+ onError={(error) => {
+ console.error('Error fetching audio:', error);
+ }}
+ id={`audio-${messageId}`}
+ muted
+ autoPlay
+ />
+ </>
+ );
+}
diff --git a/client/src/components/Audio/Voices.tsx b/client/src/components/Audio/Voices.tsx
new file mode 100644
index 0000000000..ff37efaa7d
--- /dev/null
+++ b/client/src/components/Audio/Voices.tsx
@@ -0,0 +1,94 @@
+import React from 'react';
+import { useRecoilState } from 'recoil';
+import type { Option } from '~/common';
+import DropdownNoState from '~/components/ui/DropdownNoState';
+import { useLocalize, useTTSBrowser, useTTSEdge, useTTSExternal } from '~/hooks';
+import { logger } from '~/utils';
+import store from '~/store';
+
+export function EdgeVoiceDropdown() {
+ const localize = useLocalize();
+ const { voices = [] } = useTTSEdge();
+ const [voice, setVoice] = useRecoilState(store.voice);
+
+ const handleVoiceChange = (newValue?: string | Option) => {
+ logger.log('Edge Voice changed:', newValue);
+ const newVoice = typeof newValue === 'string' ? newValue : newValue?.value;
+ if (newVoice != null) {
+ return setVoice(newVoice.toString());
+ }
+ };
+
+ return (
+ <div className="flex items-center justify-between">
+ <div>{localize('com_nav_voice_select')}</div>
+ <DropdownNoState
+ key={`edge-voice-dropdown-${voices.length}`}
+ value={voice}
+ options={voices}
+ onChange={handleVoiceChange}
+ sizeClasses="min-w-[200px] !max-w-[400px] [--anchor-max-width:400px]"
+ anchor="bottom start"
+ testId="EdgeVoiceDropdown"
+ />
+ </div>
+ );
+}
+
+export function BrowserVoiceDropdown() {
+ const localize = useLocalize();
+ const { voices = [] } = useTTSBrowser();
+ const [voice, setVoice] = useRecoilState(store.voice);
+
+ const handleVoiceChange = (newValue?: string | Option) => {
+ logger.log('Browser Voice changed:', newValue);
+ const newVoice = typeof newValue === 'string' ? newValue : newValue?.value;
+ if (newVoice != null) {
+ return setVoice(newVoice.toString());
+ }
+ };
+
+ return (
+ <div className="flex items-center justify-between">
+ <div>{localize('com_nav_voice_select')}</div>
+ <DropdownNoState
+ key={`browser-voice-dropdown-${voices.length}`}
+ value={voice}
+ options={voices}
+ onChange={handleVoiceChange}
+ sizeClasses="min-w-[200px] !max-w-[400px] [--anchor-max-width:400px]"
+ anchor="bottom start"
+ testId="BrowserVoiceDropdown"
+ />
+ </div>
+ );
+}
+
+export function ExternalVoiceDropdown() {
+ const localize = useLocalize();
+ const { voices = [] } = useTTSExternal();
+ const [voice, setVoice] = useRecoilState(store.voice);
+
+ const handleVoiceChange = (newValue?: string | Option) => {
+ logger.log('External Voice changed:', newValue);
+ const newVoice = typeof newValue === 'string' ? newValue : newValue?.value;
+ if (newVoice != null) {
+ return setVoice(newVoice.toString());
+ }
+ };
+
+ return (
+ <div className="flex items-center justify-between">
+ <div>{localize('com_nav_voice_select')}</div>
+ <DropdownNoState
+ key={`external-voice-dropdown-${voices.length}`}
+ value={voice}
+ options={voices}
+ onChange={handleVoiceChange}
+ sizeClasses="min-w-[200px] !max-w-[400px] [--anchor-max-width:400px]"
+ anchor="bottom start"
+ testId="ExternalVoiceDropdown"
+ />
+ </div>
+ );
+}
diff --git a/client/src/components/Chat/Messages/HoverButtons.tsx b/client/src/components/Chat/Messages/HoverButtons.tsx
index 938503acdb..950c4a7f9c 100644
--- a/client/src/components/Chat/Messages/HoverButtons.tsx
+++ b/client/src/components/Chat/Messages/HoverButtons.tsx
@@ -79,6 +79,7 @@ export default function HoverButtons({
messageId={message.messageId}
content={message.content ?? message.text}
isLast={isLast}
+ className="hover-button rounded-md p-1 pl-0 text-gray-500 hover:bg-gray-100 hover:text-gray-500 dark:text-gray-400/70 dark:hover:bg-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:group-hover:visible md:group-[.final-completion]:visible"
/>
)}
{isEditableEndpoint && (
diff --git a/client/src/components/Chat/Messages/MessageAudio.tsx b/client/src/components/Chat/Messages/MessageAudio.tsx
index cf76c18d25..4dd1f21fcb 100644
--- a/client/src/components/Chat/Messages/MessageAudio.tsx
+++ b/client/src/components/Chat/Messages/MessageAudio.tsx
@@ -1,104 +1,22 @@
-import { useEffect } from 'react';
+// client/src/components/Chat/Messages/MessageAudio.tsx
+import { memo } from 'react';
import { useRecoilValue } from 'recoil';
-import type { TMessageContentParts } from 'librechat-data-provider';
-import { VolumeIcon, VolumeMuteIcon, Spinner } from '~/components/svg';
-import { useLocalize, useTextToSpeech } from '~/hooks';
-import { logger } from '~/utils';
+import type { TMessageAudio } from '~/common';
+import { BrowserTTS, EdgeTTS, ExternalTTS } from '~/components/Audio/TTS';
+import { TTSEndpoints } from '~/common';
import store from '~/store';
-type THoverButtons = {
- messageId?: string;
- content?: TMessageContentParts[] | string;
- isLast: boolean;
- index: number;
-};
+function MessageAudio(props: TMessageAudio) {
+ const engineTTS = useRecoilValue<string>(store.engineTTS);
-export default function MessageAudio({ isLast, index, messageId, content }: THoverButtons) {
- const localize = useLocalize();
- const playbackRate = useRecoilValue(store.playbackRate);
-
- const { toggleSpeech, isSpeaking, isLoading, audioRef } = useTextToSpeech({
- isLast,
- index,
- messageId,
- content,
- });
-
- const renderIcon = (size: string) => {
- if (isLoading === true) {
- return <Spinner size={size} />;
- }
-
- if (isSpeaking === true) {
- return <VolumeMuteIcon size={size} />;
- }
-
- return <VolumeIcon size={size} />;
+ const TTSComponents = {
+ [TTSEndpoints.edge]: EdgeTTS,
+ [TTSEndpoints.browser]: BrowserTTS,
+ [TTSEndpoints.external]: ExternalTTS,
};
- useEffect(() => {
- const messageAudio = document.getElementById(`audio-${messageId}`) as HTMLAudioElement | null;
- if (!messageAudio) {
- return;
- }
- if (playbackRate != null && playbackRate > 0 && messageAudio.playbackRate !== playbackRate) {
- messageAudio.playbackRate = playbackRate;
- }
- }, [audioRef, isSpeaking, playbackRate, messageId]);
-
- logger.log(
- 'MessageAudio: audioRef.current?.src, audioRef.current',
- audioRef.current?.src,
- audioRef.current,
- );
-
- return (
- <>
- <button
- className="hover-button rounded-md p-1 pl-0 text-gray-500 hover:bg-gray-100 hover:text-gray-500 dark:text-gray-400/70 dark:hover:bg-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:group-hover:visible md:group-[.final-completion]:visible"
- // onMouseDownCapture={() => {
- // if (audioRef.current) {
- // audioRef.current.muted = false;
- // }
- // handleMouseDown();
- // }}
- // onMouseUpCapture={() => {
- // if (audioRef.current) {
- // audioRef.current.muted = false;
- // }
- // handleMouseUp();
- // }}
- onClickCapture={() => {
- if (audioRef.current) {
- audioRef.current.muted = false;
- }
- toggleSpeech();
- }}
- type="button"
- title={isSpeaking === true ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
- >
- {renderIcon('19')}
- </button>
- <audio
- ref={audioRef}
- controls
- preload="none"
- controlsList="nodownload nofullscreen noremoteplayback"
- style={{
- position: 'absolute',
- overflow: 'hidden',
- display: 'none',
- height: '0px',
- width: '0px',
- }}
- src={audioRef.current?.src}
- onError={(error) => {
- console.error('Error fetching audio:', error);
- }}
- id={`audio-${messageId}`}
- muted
- autoPlay
- />
- </>
- );
+ const SelectedTTS = TTSComponents[engineTTS];
+ return <SelectedTTS {...props} />;
}
+
+export default memo(MessageAudio);
diff --git a/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx b/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx
index 53a4c4686f..c2c5d4431d 100644
--- a/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx
+++ b/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx
@@ -1,37 +1,21 @@
-import React from 'react';
-import { useRecoilState, useRecoilValue } from 'recoil';
-import type { Option } from '~/common';
-import DropdownNoState from '~/components/ui/DropdownNoState';
-import { useLocalize, useTextToSpeech } from '~/hooks';
-import { logger } from '~/utils';
+import { useRecoilValue } from 'recoil';
+import {
+ EdgeVoiceDropdown,
+ BrowserVoiceDropdown,
+ ExternalVoiceDropdown,
+} from '~/components/Audio/Voices';
import store from '~/store';
+import { TTSEndpoints } from '~/common';
+
+const voiceDropdownComponentsMap = {
+ [TTSEndpoints.edge]: EdgeVoiceDropdown,
+ [TTSEndpoints.browser]: BrowserVoiceDropdown,
+ [TTSEndpoints.external]: ExternalVoiceDropdown,
+};
export default function VoiceDropdown() {
- const localize = useLocalize();
- const { voices = [] } = useTextToSpeech();
- const [voice, setVoice] = useRecoilState(store.voice);
const engineTTS = useRecoilValue<string>(store.engineTTS);
+ const VoiceDropdownComponent = voiceDropdownComponentsMap[engineTTS];
- const handleVoiceChange = (newValue?: string | Option) => {
- logger.log('Voice changed:', newValue);
- const newVoice = typeof newValue === 'string' ? newValue : newValue?.value;
- if (newVoice != null) {
- return setVoice(newVoice.toString());
- }
- };
-
- return (
- <div className="flex items-center justify-between">
- <div>{localize('com_nav_voice_select')}</div>
- <DropdownNoState
- key={`voice-dropdown-${engineTTS}-${voices.length}`}
- value={voice}
- options={voices}
- onChange={handleVoiceChange}
- sizeClasses="min-w-[200px] !max-w-[400px] [--anchor-max-width:400px]"
- anchor="bottom start"
- testId="VoiceDropdown"
- />
- </div>
- );
+ return <VoiceDropdownComponent />;
}
diff --git a/client/src/hooks/Audio/index.ts b/client/src/hooks/Audio/index.ts
index c294db3452..2252cbaceb 100644
--- a/client/src/hooks/Audio/index.ts
+++ b/client/src/hooks/Audio/index.ts
@@ -1,3 +1,6 @@
export * from './MediaSourceAppender';
export { default as useCustomAudioRef } from './useCustomAudioRef';
export { default as usePauseGlobalAudio } from './usePauseGlobalAudio';
+export { default as useTTSExternal } from './useTTSExternal';
+export { default as useTTSBrowser } from './useTTSBrowser';
+export { default as useTTSEdge } from './useTTSEdge';
diff --git a/client/src/hooks/Audio/useTTSBrowser.ts b/client/src/hooks/Audio/useTTSBrowser.ts
new file mode 100644
index 0000000000..9edd9a06ec
--- /dev/null
+++ b/client/src/hooks/Audio/useTTSBrowser.ts
@@ -0,0 +1,100 @@
+// client/src/hooks/Audio/useTTSBrowser.ts
+import { useRef, useEffect, useState } from 'react';
+import { useRecoilState, useRecoilValue } from 'recoil';
+import { parseTextParts } from 'librechat-data-provider';
+import type { TMessageContentParts } from 'librechat-data-provider';
+import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';
+import usePauseGlobalAudio from '~/hooks/Audio/usePauseGlobalAudio';
+import useAudioRef from '~/hooks/Audio/useAudioRef';
+import { logger } from '~/utils';
+import store from '~/store';
+
+type TUseTextToSpeech = {
+ messageId?: string;
+ content?: TMessageContentParts[] | string;
+ isLast?: boolean;
+ index?: number;
+};
+
+const useTTSBrowser = (props?: TUseTextToSpeech) => {
+ const { content, isLast = false, index = 0 } = props ?? {};
+
+ const isMouseDownRef = useRef(false);
+ const timerRef = useRef<number | undefined>(undefined);
+ const [isSpeakingState, setIsSpeaking] = useState(false);
+ const { audioRef } = useAudioRef({ setIsPlaying: setIsSpeaking });
+
+ const { pauseGlobalAudio } = usePauseGlobalAudio(index);
+ const [voice, setVoice] = useRecoilState(store.voice);
+ const globalIsPlaying = useRecoilValue(store.globalAudioPlayingFamily(index));
+
+ const isSpeaking = isSpeakingState || (isLast && globalIsPlaying);
+
+ const {
+ generateSpeechLocal: generateSpeech,
+ cancelSpeechLocal: cancelSpeech,
+ voices,
+ } = useTextToSpeechBrowser({ setIsSpeaking });
+
+ useEffect(() => {
+ const firstVoice = voices[0];
+ if (voices.length && typeof firstVoice === 'object') {
+ const lastSelectedVoice = voices.find((v) =>
+ typeof v === 'object' ? v.value === voice : v === voice,
+ );
+ if (lastSelectedVoice != null) {
+ const currentVoice =
+ typeof lastSelectedVoice === 'object' ? lastSelectedVoice.value : lastSelectedVoice;
+ logger.log('useTextToSpeech.ts - Effect:', { voices, voice: currentVoice });
+ setVoice(currentVoice);
+ return;
+ }
+
+ logger.log('useTextToSpeech.ts - Effect:', { voices, voice: firstVoice.value });
+ setVoice(firstVoice.value);
+ }
+ }, [setVoice, voice, voices]);
+
+ const handleMouseDown = () => {
+ isMouseDownRef.current = true;
+ timerRef.current = window.setTimeout(() => {
+ if (isMouseDownRef.current) {
+ const messageContent = content ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage);
+ }
+ }, 1000);
+ };
+
+ const handleMouseUp = () => {
+ isMouseDownRef.current = false;
+ if (timerRef.current != null) {
+ window.clearTimeout(timerRef.current);
+ }
+ };
+
+ const toggleSpeech = () => {
+ if (isSpeaking === true) {
+ cancelSpeech();
+ pauseGlobalAudio();
+ } else {
+ const messageContent = content ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage);
+ }
+ };
+
+ return {
+ handleMouseDown,
+ handleMouseUp,
+ toggleSpeech,
+ isSpeaking,
+ isLoading: false,
+ audioRef,
+ voices,
+ };
+};
+
+export default useTTSBrowser;
diff --git a/client/src/hooks/Audio/useTTSEdge.ts b/client/src/hooks/Audio/useTTSEdge.ts
new file mode 100644
index 0000000000..c757ad414f
--- /dev/null
+++ b/client/src/hooks/Audio/useTTSEdge.ts
@@ -0,0 +1,100 @@
+// client/src/hooks/Audio/useTTSEdge.ts
+import { useRef, useEffect, useState } from 'react';
+import { useRecoilState, useRecoilValue } from 'recoil';
+import { parseTextParts } from 'librechat-data-provider';
+import type { TMessageContentParts } from 'librechat-data-provider';
+import usePauseGlobalAudio from '~/hooks/Audio/usePauseGlobalAudio';
+import useTextToSpeechEdge from '~/hooks/Input/useTextToSpeechEdge';
+import useAudioRef from '~/hooks/Audio/useAudioRef';
+import { logger } from '~/utils';
+import store from '~/store';
+
+type TUseTextToSpeech = {
+ messageId?: string;
+ content?: TMessageContentParts[] | string;
+ isLast?: boolean;
+ index?: number;
+};
+
+const useTTSEdge = (props?: TUseTextToSpeech) => {
+ const { content, isLast = false, index = 0 } = props ?? {};
+
+ const isMouseDownRef = useRef(false);
+ const timerRef = useRef<number | undefined>(undefined);
+ const [isSpeakingState, setIsSpeaking] = useState(false);
+ const { audioRef } = useAudioRef({ setIsPlaying: setIsSpeaking });
+
+ const { pauseGlobalAudio } = usePauseGlobalAudio(index);
+ const [voice, setVoice] = useRecoilState(store.voice);
+ const globalIsPlaying = useRecoilValue(store.globalAudioPlayingFamily(index));
+
+ const isSpeaking = isSpeakingState || (isLast && globalIsPlaying);
+
+ const {
+ generateSpeechEdge: generateSpeech,
+ cancelSpeechEdge: cancelSpeech,
+ voices,
+ } = useTextToSpeechEdge({ setIsSpeaking });
+
+ useEffect(() => {
+ const firstVoice = voices[0];
+ if (voices.length && typeof firstVoice === 'object') {
+ const lastSelectedVoice = voices.find((v) =>
+ typeof v === 'object' ? v.value === voice : v === voice,
+ );
+ if (lastSelectedVoice != null) {
+ const currentVoice =
+ typeof lastSelectedVoice === 'object' ? lastSelectedVoice.value : lastSelectedVoice;
+ logger.log('useTextToSpeech.ts - Effect:', { voices, voice: currentVoice });
+ setVoice(currentVoice);
+ return;
+ }
+
+ logger.log('useTextToSpeech.ts - Effect:', { voices, voice: firstVoice.value });
+ setVoice(firstVoice.value);
+ }
+ }, [setVoice, voice, voices]);
+
+ const handleMouseDown = () => {
+ isMouseDownRef.current = true;
+ timerRef.current = window.setTimeout(() => {
+ if (isMouseDownRef.current) {
+ const messageContent = content ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage);
+ }
+ }, 1000);
+ };
+
+ const handleMouseUp = () => {
+ isMouseDownRef.current = false;
+ if (timerRef.current != null) {
+ window.clearTimeout(timerRef.current);
+ }
+ };
+
+ const toggleSpeech = () => {
+ if (isSpeaking === true) {
+ cancelSpeech();
+ pauseGlobalAudio();
+ } else {
+ const messageContent = content ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage);
+ }
+ };
+
+ return {
+ handleMouseDown,
+ handleMouseUp,
+ toggleSpeech,
+ isSpeaking,
+ isLoading: false,
+ audioRef,
+ voices,
+ };
+};
+
+export default useTTSEdge;
diff --git a/client/src/hooks/Audio/useTTSExternal.ts b/client/src/hooks/Audio/useTTSExternal.ts
new file mode 100644
index 0000000000..d296abf4a4
--- /dev/null
+++ b/client/src/hooks/Audio/useTTSExternal.ts
@@ -0,0 +1,101 @@
+// client/src/hooks/Audio/useTTSExternal.ts
+import { useRef, useEffect, useState } from 'react';
+import { useRecoilState, useRecoilValue } from 'recoil';
+import { parseTextParts } from 'librechat-data-provider';
+import type { TMessageContentParts } from 'librechat-data-provider';
+import useTextToSpeechExternal from '~/hooks/Input/useTextToSpeechExternal';
+import usePauseGlobalAudio from '~/hooks/Audio/usePauseGlobalAudio';
+import useAudioRef from '~/hooks/Audio/useAudioRef';
+import { logger } from '~/utils';
+import store from '~/store';
+
+type TUseTextToSpeech = {
+ messageId?: string;
+ content?: TMessageContentParts[] | string;
+ isLast?: boolean;
+ index?: number;
+};
+
+const useTTSExternal = (props?: TUseTextToSpeech) => {
+ const { messageId, content, isLast = false, index = 0 } = props ?? {};
+
+ const isMouseDownRef = useRef(false);
+ const timerRef = useRef<number | undefined>(undefined);
+ const [isSpeakingState, setIsSpeaking] = useState(false);
+ const { audioRef } = useAudioRef({ setIsPlaying: setIsSpeaking });
+
+ const { pauseGlobalAudio } = usePauseGlobalAudio(index);
+ const [voice, setVoice] = useRecoilState(store.voice);
+ const globalIsPlaying = useRecoilValue(store.globalAudioPlayingFamily(index));
+
+ const isSpeaking = isSpeakingState || (isLast && globalIsPlaying);
+ const {
+ cancelSpeech,
+ generateSpeechExternal: generateSpeech,
+ isLoading,
+ voices,
+ } = useTextToSpeechExternal({
+ setIsSpeaking,
+ audioRef,
+ messageId,
+ isLast,
+ index,
+ });
+
+ useEffect(() => {
+ const firstVoice = voices[0];
+ if (voices.length) {
+ const lastSelectedVoice = voices.find((v) => v === voice);
+ if (lastSelectedVoice != null) {
+ logger.log('useTextToSpeech.ts - Effect:', { voices, voice: lastSelectedVoice });
+ setVoice(lastSelectedVoice.toString());
+ return;
+ }
+ logger.log('useTextToSpeech.ts - Effect:', { voices, voice: firstVoice });
+ setVoice(firstVoice.toString());
+ }
+ }, [setVoice, voice, voices]);
+
+ const handleMouseDown = () => {
+ isMouseDownRef.current = true;
+ timerRef.current = window.setTimeout(() => {
+ if (isMouseDownRef.current) {
+ const messageContent = content ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage, false);
+ }
+ }, 1000);
+ };
+
+ const handleMouseUp = () => {
+ isMouseDownRef.current = false;
+ if (timerRef.current != null) {
+ window.clearTimeout(timerRef.current);
+ }
+ };
+
+ const toggleSpeech = () => {
+ if (isSpeaking === true) {
+ cancelSpeech();
+ pauseGlobalAudio();
+ } else {
+ const messageContent = content ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage, false);
+ }
+ };
+
+ return {
+ handleMouseDown,
+ handleMouseUp,
+ toggleSpeech,
+ isSpeaking,
+ isLoading,
+ audioRef,
+ voices,
+ };
+};
+
+export default useTTSExternal;
diff --git a/client/src/hooks/Input/useGetAudioSettings.ts b/client/src/hooks/Input/useGetAudioSettings.ts
index 99e8854f21..899cf50217 100644
--- a/client/src/hooks/Input/useGetAudioSettings.ts
+++ b/client/src/hooks/Input/useGetAudioSettings.ts
@@ -2,17 +2,6 @@ import { useMemo } from 'react';
import { useRecoilValue } from 'recoil';
import store from '~/store';
-export enum STTEndpoints {
- browser = 'browser',
- external = 'external',
-}
-
-export enum TTSEndpoints {
- browser = 'browser',
- edge = 'edge',
- external = 'external',
-}
-
const useGetAudioSettings = () => {
const engineSTT = useRecoilValue<string>(store.engineSTT);
const engineTTS = useRecoilValue<string>(store.engineTTS);
diff --git a/client/src/hooks/Input/useTextToSpeech.ts b/client/src/hooks/Input/useTextToSpeech.ts
index 680651110c..4d142b432c 100644
--- a/client/src/hooks/Input/useTextToSpeech.ts
+++ b/client/src/hooks/Input/useTextToSpeech.ts
@@ -3,10 +3,10 @@ import { useRef, useMemo, useEffect, useState } from 'react';
import { parseTextParts } from 'librechat-data-provider';
import type { TMessageContentParts } from 'librechat-data-provider';
import type { Option } from '~/common';
-import useTextToSpeechExternal from './useTextToSpeechExternal';
-import useTextToSpeechBrowser from './useTextToSpeechBrowser';
-import useGetAudioSettings from './useGetAudioSettings';
-import useTextToSpeechEdge from './useTextToSpeechEdge';
+import useTextToSpeechExternal from '~/hooks/Input/useTextToSpeechExternal';
+import useTextToSpeechBrowser from '~/hooks/Input/useTextToSpeechBrowser';
+import useGetAudioSettings from '~/hooks/Input/useGetAudioSettings';
+import useTextToSpeechEdge from '~/hooks/Input/useTextToSpeechEdge';
import useAudioRef from '~/hooks/Audio/useAudioRef';
import { usePauseGlobalAudio } from '../Audio';
import { logger } from '~/utils';
diff --git a/client/src/hooks/Input/useTextToSpeechBrowser.ts b/client/src/hooks/Input/useTextToSpeechBrowser.ts
index 8446a7b3b8..85c3a80c00 100644
--- a/client/src/hooks/Input/useTextToSpeechBrowser.ts
+++ b/client/src/hooks/Input/useTextToSpeechBrowser.ts
@@ -1,43 +1,54 @@
import { useRecoilState } from 'recoil';
import { useState, useEffect, useCallback } from 'react';
+import type { VoiceOption } from '~/common';
import store from '~/store';
-interface VoiceOption {
- value: string;
- label: string;
-}
-
function useTextToSpeechBrowser({
setIsSpeaking,
}: {
- setIsSpeaking: (isSpeaking: boolean) => void;
+ setIsSpeaking: React.Dispatch<React.SetStateAction<boolean>>;
}) {
const [cloudBrowserVoices] = useRecoilState(store.cloudBrowserVoices);
const [voiceName] = useRecoilState(store.voice);
const [voices, setVoices] = useState<VoiceOption[]>([]);
const updateVoices = useCallback(() => {
- const availableVoices = window.speechSynthesis
- .getVoices()
- .filter((v) => cloudBrowserVoices || v.localService === true);
+ try {
+ const availableVoices = window.speechSynthesis.getVoices();
+ if (!Array.isArray(availableVoices)) {
+ console.error('getVoices() did not return an array');
+ return;
+ }
- const voiceOptions: VoiceOption[] = availableVoices.map((v) => ({
- value: v.name,
- label: v.name,
- }));
+ const filteredVoices = availableVoices.filter(
+ (v) => cloudBrowserVoices || v.localService === true,
+ );
+ const voiceOptions: VoiceOption[] = filteredVoices.map((v) => ({
+ value: v.name,
+ label: v.name,
+ }));
- setVoices(voiceOptions);
+ setVoices(voiceOptions);
+ } catch (error) {
+ console.error('Error updating voices:', error);
+ }
}, [cloudBrowserVoices]);
useEffect(() => {
- if (window.speechSynthesis.getVoices().length) {
- updateVoices();
- } else {
- window.speechSynthesis.onvoiceschanged = updateVoices;
+ const synth = window.speechSynthesis;
+
+ try {
+ if (synth.getVoices().length) {
+ updateVoices();
+ } else {
+ synth.onvoiceschanged = updateVoices;
+ }
+ } catch (error) {
+ console.error('Error in useEffect:', error);
}
return () => {
- window.speechSynthesis.onvoiceschanged = null;
+ synth.onvoiceschanged = null;
};
}, [updateVoices]);
@@ -46,22 +57,37 @@ function useTextToSpeechBrowser({
const voice = voices.find((v) => v.value === voiceName);
if (!voice) {
+ console.warn('Selected voice not found');
return;
}
- synth.cancel();
- const utterance = new SpeechSynthesisUtterance(text);
- utterance.voice = synth.getVoices().find((v) => v.name === voice.value) || null;
- utterance.onend = () => {
+ try {
+ synth.cancel();
+ const utterance = new SpeechSynthesisUtterance(text);
+ utterance.voice = synth.getVoices().find((v) => v.name === voice.value) || null;
+ utterance.onend = () => {
+ setIsSpeaking(false);
+ };
+ utterance.onerror = (event) => {
+ console.error('Speech synthesis error:', event);
+ setIsSpeaking(false);
+ };
+ setIsSpeaking(true);
+ synth.speak(utterance);
+ } catch (error) {
+ console.error('Error generating speech:', error);
setIsSpeaking(false);
- };
- setIsSpeaking(true);
- synth.speak(utterance);
+ }
};
const cancelSpeechLocal = () => {
- window.speechSynthesis.cancel();
- setIsSpeaking(false);
+ try {
+ window.speechSynthesis.cancel();
+ } catch (error) {
+ console.error('Error cancelling speech:', error);
+ } finally {
+ setIsSpeaking(false);
+ }
};
return { generateSpeechLocal, cancelSpeechLocal, voices };
diff --git a/client/src/hooks/Input/useTextToSpeechEdge.ts b/client/src/hooks/Input/useTextToSpeechEdge.ts
index 50ee976b66..ab92618f52 100644
--- a/client/src/hooks/Input/useTextToSpeechEdge.ts
+++ b/client/src/hooks/Input/useTextToSpeechEdge.ts
@@ -1,28 +1,24 @@
import { useRecoilValue } from 'recoil';
-import { useState, useCallback, useRef, useEffect, useMemo } from 'react';
import { MsEdgeTTS, OUTPUT_FORMAT } from 'msedge-tts';
-import { useToastContext } from '~/Providers';
+import { useState, useCallback, useRef, useEffect, useMemo } from 'react';
+import type { VoiceOption } from '~/common';
+import { useToastContext } from '~/Providers/ToastContext';
import useLocalize from '~/hooks/useLocalize';
import store from '~/store';
-interface Voice {
- value: string;
- label: string;
-}
-
interface UseTextToSpeechEdgeReturn {
generateSpeechEdge: (text: string) => void;
cancelSpeechEdge: () => void;
- voices: Voice[];
+ voices: VoiceOption[];
}
function useTextToSpeechEdge({
setIsSpeaking,
}: {
- setIsSpeaking: (isSpeaking: boolean) => void;
+ setIsSpeaking: React.Dispatch<React.SetStateAction<boolean>>;
}): UseTextToSpeechEdgeReturn {
const localize = useLocalize();
- const [voices, setVoices] = useState<Voice[]>([]);
+ const [voices, setVoices] = useState<VoiceOption[]>([]);
const voiceName = useRecoilValue(store.voice);
const ttsRef = useRef<MsEdgeTTS | null>(null);
const audioElementRef = useRef<HTMLAudioElement | null>(null);
@@ -63,7 +59,7 @@ function useTextToSpeechEdge({
if (!ttsRef.current) {
ttsRef.current = new MsEdgeTTS();
}
- const availableVoice: Voice | undefined = voices.find((v) => v.value === voiceName);
+ const availableVoice: VoiceOption | undefined = voices.find((v) => v.value === voiceName);
if (availableVoice) {
ttsRef.current
@@ -181,7 +177,7 @@ function useTextToSpeechEdge({
generate();
},
- [appendNextBuffer, showToast, localize],
+ [setIsSpeaking, appendNextBuffer, showToast, localize],
);
const cancelSpeechEdge = useCallback(() => {
@@ -202,7 +198,7 @@ function useTextToSpeechEdge({
status: 'error',
});
}
- }, [showToast, localize]);
+ }, [setIsSpeaking, showToast, localize]);
useEffect(() => {
if (!isBrowserSupported) {
diff --git a/client/src/hooks/Input/useTextToSpeechExternal.ts b/client/src/hooks/Input/useTextToSpeechExternal.ts
index fc48e11ae1..8e616e4cde 100644
--- a/client/src/hooks/Input/useTextToSpeechExternal.ts
+++ b/client/src/hooks/Input/useTextToSpeechExternal.ts
@@ -1,8 +1,8 @@
import { useRecoilValue } from 'recoil';
import { useState, useMemo, useRef, useCallback, useEffect } from 'react';
import { useTextToSpeechMutation, useVoicesQuery } from '~/data-provider';
+import { useToastContext } from '~/Providers/ToastContext';
import useLocalize from '~/hooks/useLocalize';
-import { useToastContext } from '~/Providers';
import store from '~/store';
const createFormData = (text: string, voice: string) => {
@@ -13,7 +13,7 @@ const createFormData = (text: string, voice: string) => {
};
type TUseTTSExternal = {
- setIsSpeaking: (isSpeaking: boolean) => void;
+ setIsSpeaking: React.Dispatch<React.SetStateAction<boolean>>;
audioRef: React.MutableRefObject<HTMLAudioElement | null>;
messageId?: string;
isLast: boolean;
diff --git a/client/src/hooks/index.ts b/client/src/hooks/index.ts
index 507c12ed59..a052b82f58 100644
--- a/client/src/hooks/index.ts
+++ b/client/src/hooks/index.ts
@@ -1,3 +1,4 @@
+export * from './Audio';
export * from './Assistants';
export * from './Chat';
export * from './Config';
diff --git a/client/src/localization/languages/Eng.ts b/client/src/localization/languages/Eng.ts
index 8c39d5db4d..34cbe3dcb6 100644
--- a/client/src/localization/languages/Eng.ts
+++ b/client/src/localization/languages/Eng.ts
@@ -664,6 +664,8 @@ export default {
com_nav_audio_process_error: 'Error processing audio: {0}',
com_nav_long_audio_warning: 'Longer texts will take longer to process.',
com_nav_tts_init_error: 'Failed to initialize text-to-speech: {0}',
+ com_nav_tts_unsupported_error:
+ 'Text-to-speech for the selected engine is not supported in this browser.',
com_nav_source_buffer_error: 'Error setting up audio playback. Please refresh the page.',
com_nav_media_source_init_error:
'Unable to prepare audio player. Please check your browser settings.',
diff --git a/librechat.example.yaml b/librechat.example.yaml
index 009d757ae9..c5b69bf9b7 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -31,7 +31,8 @@ registration:
# url: ''
# apiKey: '${TTS_API_KEY}'
# model: ''
-# voice: ''
+# voices: ['']
+
#
# stt:
# openai: