Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔊 fix(tts): NotAllowedError (mobile/safari), Unsupported MediaSource type (firefox), Hide Audio Element (#2854)

Danny Avila35ba4ba1a410
diff --git a/api/cache/redis.js b/api/cache/redis.js
index c9bfdd6cae..adf291d02b 100644
--- a/api/cache/redis.js
+++ b/api/cache/redis.js
@@ -1,9 +1,4 @@
const Redis = require('ioredis');
-const { logger } = require('~/config');
const { REDIS_URI } = process.env ?? {};
-const redis = new Redis(REDIS_URI);
-redis
- .on('error', (err) => logger.error('ioredis error:', err))
- .on('ready', () => logger.info('ioredis successfully initialized.'))
- .on('reconnecting', () => logger.info('ioredis reconnecting...'));
+const redis = new Redis.Cluster(REDIS_URI);
module.exports = redis;
diff --git a/api/server/services/Files/Audio/textToSpeech.js b/api/server/services/Files/Audio/textToSpeech.js
index 2d77324ce4..ab7a5e55dc 100644
--- a/api/server/services/Files/Audio/textToSpeech.js
+++ b/api/server/services/Files/Audio/textToSpeech.js
@@ -16,7 +16,8 @@ const { logger } = require('~/config');
function getProvider(ttsSchema) {
if (!ttsSchema) {
throw new Error(`No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)?
-# Example TTS configuration`);
+
+ https://www.librechat.ai/docs/configuration/stt_tts#tts`);
}
const providers = Object.entries(ttsSchema).filter(([, value]) => Object.keys(value).length > 0);
diff --git a/api/server/socialLogins.js b/api/server/socialLogins.js
index 4abe278b84..66ee5f9e42 100644
--- a/api/server/socialLogins.js
+++ b/api/server/socialLogins.js
@@ -1,14 +1,15 @@
+const Redis = require('ioredis');
+const passport = require('passport');
const session = require('express-session');
const RedisStore = require('connect-redis').default;
-const passport = require('passport');
const {
+ setupOpenId,
googleLogin,
githubLogin,
discordLogin,
facebookLogin,
- setupOpenId,
-} = require('../strategies');
-const client = require('../cache/redis');
+} = require('~/strategies');
+const { logger } = require('~/config');
/**
*
@@ -40,6 +41,11 @@ const configureSocialLogins = (app) => {
saveUninitialized: false,
};
if (process.env.USE_REDIS) {
+ const client = new Redis(process.env.REDIS_URI);
+ client
+ .on('error', (err) => logger.error('ioredis error:', err))
+ .on('ready', () => logger.info('ioredis successfully initialized.'))
+ .on('reconnecting', () => logger.info('ioredis reconnecting...'));
sessionOptions.store = new RedisStore({ client, prefix: 'librechat' });
}
app.use(session(sessionOptions));
diff --git a/client/src/components/Chat/Input/AudioRecorder.tsx b/client/src/components/Chat/Input/AudioRecorder.tsx
index c1d100e50d..d4ea2c4a8e 100644
--- a/client/src/components/Chat/Input/AudioRecorder.tsx
+++ b/client/src/components/Chat/Input/AudioRecorder.tsx
@@ -1,16 +1,46 @@
-import React from 'react';
-import { ListeningIcon, Spinner, SpeechIcon } from '~/components/svg';
+import { useEffect } from 'react';
+import type { UseFormReturn } from 'react-hook-form';
import { TooltipProvider, Tooltip, TooltipTrigger, TooltipContent } from '~/components/ui/';
-import { useLocalize } from '~/hooks';
+import { ListeningIcon, Spinner } from '~/components/svg';
+import { useLocalize, useSpeechToText } from '~/hooks';
+import { globalAudioId } from '~/common';
export default function AudioRecorder({
- isListening,
- isLoading,
- startRecording,
- stopRecording,
+ textAreaRef,
+ methods,
+ ask,
disabled,
+}: {
+ textAreaRef: React.RefObject<HTMLTextAreaElement>;
+ methods: UseFormReturn<{ text: string }>;
+ ask: (data: { text: string }) => void;
+ disabled: boolean;
}) {
const localize = useLocalize();
+
+ const handleTranscriptionComplete = (text: string) => {
+ if (text) {
+ const globalAudio = document.getElementById(globalAudioId) as HTMLAudioElement;
+ if (globalAudio) {
+ console.log('Unmuting global audio');
+ globalAudio.muted = false;
+ }
+ ask({ text });
+ methods.reset({ text: '' });
+ clearText();
+ }
+ };
+
+ const { isListening, isLoading, startRecording, stopRecording, speechText, clearText } =
+ useSpeechToText(handleTranscriptionComplete);
+
+ useEffect(() => {
+ if (textAreaRef.current) {
+ textAreaRef.current.value = speechText;
+ methods.setValue('text', speechText, { shouldValidate: true });
+ }
+ }, [speechText, methods, textAreaRef]);
+
const handleStartRecording = async () => {
await startRecording();
};
@@ -19,6 +49,16 @@ export default function AudioRecorder({
await stopRecording();
};
+ const renderIcon = () => {
+ if (isListening) {
+ return <ListeningIcon className="stroke-red-500" />;
+ }
+ if (isLoading) {
+ return <Spinner className="stroke-gray-700 dark:stroke-gray-300" />;
+ }
+ return <ListeningIcon className="stroke-gray-700 dark:stroke-gray-300" />;
+ };
+
return (
<TooltipProvider delayDuration={250}>
<Tooltip>
@@ -29,13 +69,7 @@ export default function AudioRecorder({
className="absolute bottom-1.5 right-12 flex h-[30px] w-[30px] items-center justify-center rounded-lg p-0.5 transition-colors hover:bg-gray-200 dark:hover:bg-gray-700 md:bottom-3 md:right-12"
type="button"
>
- {isListening ? (
- <SpeechIcon className="stroke-gray-700 dark:stroke-gray-300" />
- ) : isLoading ? (
- <Spinner className="stroke-gray-700 dark:stroke-gray-300" />
- ) : (
- <ListeningIcon className="stroke-gray-700 dark:stroke-gray-300" />
- )}
+ {renderIcon()}
</button>
</TooltipTrigger>
<TooltipContent side="top" sideOffset={10}>
diff --git a/client/src/components/Chat/Input/ChatForm.tsx b/client/src/components/Chat/Input/ChatForm.tsx
index d3cb04e422..f5cb4232ca 100644
--- a/client/src/components/Chat/Input/ChatForm.tsx
+++ b/client/src/components/Chat/Input/ChatForm.tsx
@@ -1,6 +1,6 @@
import { useForm } from 'react-hook-form';
import { useRecoilState, useRecoilValue } from 'recoil';
-import { memo, useCallback, useRef, useMemo, useEffect } from 'react';
+import { memo, useCallback, useRef, useMemo } from 'react';
import {
supportsFiles,
mergeFileConfig,
@@ -8,7 +8,7 @@ import {
fileConfig as defaultFileConfig,
} from 'librechat-data-provider';
import { useChatContext, useAssistantsMapContext } from '~/Providers';
-import { useRequiresKey, useTextarea, useSpeechToText } from '~/hooks';
+import { useRequiresKey, useTextarea } from '~/hooks';
import { TextareaAutosize } from '~/components/ui';
import { useGetFileConfig } from '~/data-provider';
import { cn, removeFocusOutlines } from '~/utils';
@@ -72,24 +72,6 @@ const ChatForm = ({ index = 0 }) => {
const { endpoint: _endpoint, endpointType } = conversation ?? { endpoint: null };
const endpoint = endpointType ?? _endpoint;
- const handleTranscriptionComplete = (text: string) => {
- if (text) {
- ask({ text });
- methods.reset({ text: '' });
- clearText();
- }
- };
-
- const { isListening, isLoading, startRecording, stopRecording, speechText, clearText } =
- useSpeechToText(handleTranscriptionComplete);
-
- useEffect(() => {
- if (textAreaRef.current) {
- textAreaRef.current.value = speechText;
- methods.setValue('text', speechText, { shouldValidate: true });
- }
- }, [speechText, methods]);
-
const { data: fileConfig = defaultFileConfig } = useGetFileConfig({
select: (data) => mergeFileConfig(data),
});
@@ -183,11 +165,10 @@ const ChatForm = ({ index = 0 }) => {
)}
{SpeechToText && (
<AudioRecorder
- isListening={isListening}
- isLoading={isLoading}
- startRecording={startRecording}
- stopRecording={stopRecording}
disabled={!!disableInputs}
+ textAreaRef={textAreaRef}
+ ask={submitMessage}
+ methods={methods}
/>
)}
{TextToSpeech && automaticPlayback && <StreamAudio index={index} />}
diff --git a/client/src/components/Chat/Input/StreamAudio.tsx b/client/src/components/Chat/Input/StreamAudio.tsx
index b89c295d37..ebf0f4e48e 100644
--- a/client/src/components/Chat/Input/StreamAudio.tsx
+++ b/client/src/components/Chat/Input/StreamAudio.tsx
@@ -88,7 +88,7 @@ export default function StreamAudio({ index = 0 }) {
return;
}
- console.log('Fetching audio...');
+ console.log('Fetching audio...', navigator.userAgent);
const response = await fetch('/api/files/tts', {
method: 'POST',
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` },
@@ -103,8 +103,14 @@ export default function StreamAudio({ index = 0 }) {
}
const reader = response.body.getReader();
- const mediaSource = new MediaSourceAppender('audio/mpeg');
- setGlobalAudioURL(mediaSource.mediaSourceUrl);
+
+ const type = 'audio/mpeg';
+ const browserSupportsType = MediaSource.isTypeSupported(type);
+ let mediaSource: MediaSourceAppender | undefined;
+ if (browserSupportsType) {
+ mediaSource = new MediaSourceAppender(type);
+ setGlobalAudioURL(mediaSource.mediaSourceUrl);
+ }
setAudioRunId(activeRunId);
let done = false;
@@ -120,7 +126,7 @@ export default function StreamAudio({ index = 0 }) {
if (cacheTTS && value) {
chunks.push(value);
}
- if (value) {
+ if (value && mediaSource) {
mediaSource.addData(value);
}
done = readerDone;
@@ -136,8 +142,19 @@ export default function StreamAudio({ index = 0 }) {
if (!cacheKey) {
throw new Error('Cache key not found');
}
- const audioBlob = new Blob(chunks, { type: 'audio/mpeg' });
- cache.put(cacheKey, new Response(audioBlob));
+ const audioBlob = new Blob(chunks, { type });
+ const cachedResponse = new Response(audioBlob);
+ await cache.put(cacheKey, cachedResponse);
+ if (!browserSupportsType) {
+ const unconsumedResponse = await cache.match(cacheKey);
+ if (!unconsumedResponse) {
+ throw new Error('Failed to fetch audio from cache');
+ }
+ const audioBlob = await unconsumedResponse.blob();
+ const blobUrl = URL.createObjectURL(audioBlob);
+ setGlobalAudioURL(blobUrl);
+ }
+ setIsFetching(false);
}
console.log('Audio stream reading ended');
@@ -194,9 +211,16 @@ export default function StreamAudio({ index = 0 }) {
ref={audioRef}
controls
controlsList="nodownload nofullscreen noremoteplayback"
- className="absolute h-0 w-0 overflow-hidden"
+ style={{
+ position: 'absolute',
+ overflow: 'hidden',
+ display: 'none',
+ height: '0px',
+ width: '0px',
+ }}
src={globalAudioURL || undefined}
id={globalAudioId}
+ muted
autoPlay
/>
);
diff --git a/client/src/components/Chat/Messages/HoverButtons.tsx b/client/src/components/Chat/Messages/HoverButtons.tsx
index ab9052dc77..02e4414818 100644
--- a/client/src/components/Chat/Messages/HoverButtons.tsx
+++ b/client/src/components/Chat/Messages/HoverButtons.tsx
@@ -1,18 +1,10 @@
import React, { useState } from 'react';
import { useRecoilState } from 'recoil';
import type { TConversation, TMessage } from 'librechat-data-provider';
-import {
- Clipboard,
- CheckMark,
- EditIcon,
- RegenerateIcon,
- ContinueIcon,
- VolumeIcon,
- VolumeMuteIcon,
- Spinner,
-} from '~/components/svg';
-import { useGenerationsByLatest, useLocalize, useTextToSpeech } from '~/hooks';
+import { EditIcon, Clipboard, CheckMark, ContinueIcon, RegenerateIcon } from '~/components/svg';
+import { useGenerationsByLatest, useLocalize } from '~/hooks';
import { Fork } from '~/components/Conversations';
+import MessageAudio from './MessageAudio';
import { cn } from '~/utils';
import store from '~/store';
@@ -49,12 +41,6 @@ export default function HoverButtons({
const [isCopied, setIsCopied] = useState(false);
const [TextToSpeech] = useRecoilState<boolean>(store.TextToSpeech);
- const { handleMouseDown, handleMouseUp, toggleSpeech, isSpeaking, isLoading } = useTextToSpeech(
- message?.content ?? message?.text ?? '',
- isLast,
- index,
- );
-
const {
hideEditButton,
regenerateEnabled,
@@ -81,32 +67,9 @@ export default function HoverButtons({
enterEdit();
};
- const renderIcon = (size: string) => {
- if (isLoading) {
- return <Spinner size={size} />;
- }
-
- if (isSpeaking) {
- return <VolumeMuteIcon size={size} />;
- }
-
- return <VolumeIcon size={size} />;
- };
-
return (
<div className="visible mt-0 flex justify-center gap-1 self-end text-gray-400 lg:justify-start">
- {TextToSpeech && (
- <button
- className="hover-button rounded-md p-1 pl-0 text-gray-400 hover:text-gray-950 dark:text-gray-400/70 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:group-hover:visible md:group-[.final-completion]:visible"
- onMouseDown={handleMouseDown}
- onMouseUp={handleMouseUp}
- onClick={toggleSpeech}
- type="button"
- title={isSpeaking ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
- >
- {renderIcon('19')}
- </button>
- )}
+ {TextToSpeech && <MessageAudio index={index} message={message} isLast={isLast} />}
{isEditableEndpoint && (
<button
className={cn(
diff --git a/client/src/components/Chat/Messages/MessageAudio.tsx b/client/src/components/Chat/Messages/MessageAudio.tsx
new file mode 100644
index 0000000000..3c76790e38
--- /dev/null
+++ b/client/src/components/Chat/Messages/MessageAudio.tsx
@@ -0,0 +1,89 @@
+import { useEffect } from 'react';
+import { useRecoilValue } from 'recoil';
+import type { TMessage } from 'librechat-data-provider';
+import { VolumeIcon, VolumeMuteIcon, Spinner } from '~/components/svg';
+import { useLocalize, useTextToSpeech } from '~/hooks';
+import store from '~/store';
+
+type THoverButtons = {
+ message: TMessage;
+ isLast: boolean;
+ index: number;
+};
+
+export default function MessageAudio({ index, message, isLast }: THoverButtons) {
+ const localize = useLocalize();
+ const playbackRate = useRecoilValue(store.playbackRate);
+
+ const { toggleSpeech, isSpeaking, isLoading, audioRef } = useTextToSpeech(message, isLast, index);
+
+ const renderIcon = (size: string) => {
+ if (isLoading) {
+ return <Spinner size={size} />;
+ }
+
+ if (isSpeaking) {
+ return <VolumeMuteIcon size={size} />;
+ }
+
+ return <VolumeIcon size={size} />;
+ };
+
+ useEffect(() => {
+ const messageAudio = document.getElementById(
+ `audio-${message.messageId}`,
+ ) as HTMLAudioElement | null;
+ if (!messageAudio) {
+ return;
+ }
+ if (playbackRate && messageAudio && messageAudio.playbackRate !== playbackRate) {
+ messageAudio.playbackRate = playbackRate;
+ }
+ }, [audioRef, isSpeaking, playbackRate, message.messageId]);
+
+ return (
+ <>
+ <button
+ className="hover-button rounded-md p-1 pl-0 text-gray-400 hover:text-gray-950 dark:text-gray-400/70 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:group-hover:visible md:group-[.final-completion]:visible"
+ // onMouseDownCapture={() => {
+ // if (audioRef.current) {
+ // audioRef.current.muted = false;
+ // }
+ // handleMouseDown();
+ // }}
+ // onMouseUpCapture={() => {
+ // if (audioRef.current) {
+ // audioRef.current.muted = false;
+ // }
+ // handleMouseUp();
+ // }}
+ onClickCapture={() => {
+ if (audioRef.current) {
+ audioRef.current.muted = false;
+ }
+ toggleSpeech();
+ }}
+ type="button"
+ title={isSpeaking ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
+ >
+ {renderIcon('19')}
+ </button>
+ <audio
+ ref={audioRef}
+ controls
+ controlsList="nodownload nofullscreen noremoteplayback"
+ style={{
+ position: 'absolute',
+ overflow: 'hidden',
+ display: 'none',
+ height: '0px',
+ width: '0px',
+ }}
+ src={audioRef.current?.src || undefined}
+ id={`audio-${message.messageId}`}
+ muted
+ autoPlay
+ />
+ </>
+ );
+}
diff --git a/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx b/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx
index 8037ed4086..f19ee6e55a 100644
--- a/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx
+++ b/client/src/components/Nav/SettingsTabs/Speech/TTS/VoiceDropdown.tsx
@@ -1,15 +1,21 @@
-import { useMemo } from 'react';
import { useRecoilState } from 'recoil';
+import { useMemo, useEffect } from 'react';
+import Dropdown from '~/components/ui/DropdownNoState';
import { useVoicesQuery } from '~/data-provider';
-import { Dropdown } from '~/components/ui';
import { useLocalize } from '~/hooks';
import store from '~/store';
export default function VoiceDropdown() {
const localize = useLocalize();
- const [voice, setVoice] = useRecoilState<string>(store.voice);
+ const [voice, setVoice] = useRecoilState(store.voice);
const { data } = useVoicesQuery();
+ useEffect(() => {
+ if (!voice && data?.length) {
+ setVoice(data[0]);
+ }
+ }, [voice, data, setVoice]);
+
const voiceOptions = useMemo(
() => (data ?? []).map((v: string) => ({ value: v, display: v })),
[data],
diff --git a/client/src/components/ui/DropdownNoState.tsx b/client/src/components/ui/DropdownNoState.tsx
new file mode 100644
index 0000000000..8f05394b0c
--- /dev/null
+++ b/client/src/components/ui/DropdownNoState.tsx
@@ -0,0 +1,105 @@
+import React, { FC } from 'react';
+import { Listbox } from '@headlessui/react';
+import { cn } from '~/utils/';
+
+type OptionType = {
+ value: string;
+ display?: string;
+};
+
+type DropdownPosition = 'left' | 'right';
+
+interface DropdownProps {
+ value: string;
+ label?: string;
+ onChange: (value: string) => void;
+ options: (string | OptionType)[];
+ className?: string;
+ position?: DropdownPosition;
+ width?: number;
+ maxHeight?: string;
+ testId?: string;
+}
+
+const Dropdown: FC<DropdownProps> = ({
+ value,
+ label = '',
+ onChange,
+ options,
+ className = '',
+ position = 'right',
+ width,
+ maxHeight = 'auto',
+ testId = 'dropdown-menu',
+}) => {
+ const positionClasses = {
+ right: 'origin-bottom-left left-0',
+ left: 'origin-bottom-right right-0',
+ };
+
+ return (
+ <div className={cn('relative', className)}>
+ <Listbox
+ value={value}
+ onChange={(newValue) => {
+ onChange(newValue);
+ }}
+ >
+ <div className={cn('relative', className)}>
+ <Listbox.Button
+ data-testid={testId}
+ className={cn(
+ 'relative inline-flex items-center justify-between rounded-md border-gray-300 bg-white py-2 pl-3 pr-8 text-gray-700 hover:bg-gray-50 dark:border-gray-600 dark:bg-gray-700 dark:text-white dark:hover:bg-gray-600',
+ 'w-auto',
+ className,
+ )}
+ >
+ <span className="block truncate">
+ {label}
+ {options
+ .map((o) => (typeof o === 'string' ? { value: o, display: o } : o))
+ .find((o) => o.value === value)?.display || value}
+ </span>
+ <span className="pointer-events-none absolute inset-y-0 right-0 flex items-center pr-2">
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ fill="none"
+ viewBox="0 0 24 24"
+ strokeWidth="2"
+ stroke="currentColor"
+ className="h-4 w-5 rotate-0 transform text-gray-400 transition-transform duration-300 ease-in-out"
+ >
+ <polyline points="6 9 12 15 18 9"></polyline>
+ </svg>
+ </span>
+ </Listbox.Button>
+ <Listbox.Options
+ className={cn(
+ `absolute z-50 mt-1 flex max-h-[40vh] flex-col items-start gap-1 overflow-auto rounded-lg border border-gray-300 bg-white p-1.5 text-gray-700 shadow-lg transition-opacity focus:outline-none dark:border-gray-600 dark:bg-gray-700 dark:text-white ${positionClasses[position]}`,
+ className,
+ )}
+ style={{ width: width ? `${width}px` : 'auto', maxHeight: maxHeight }}
+ >
+ {options.map((item, index) => (
+ <Listbox.Option
+ key={index}
+ value={typeof item === 'string' ? item : item.value}
+ className={cn(
+ 'relative cursor-pointer select-none rounded border-gray-300 bg-white py-2.5 pl-3 pr-6 text-gray-700 hover:bg-gray-100 dark:border-gray-300 dark:bg-gray-700 dark:text-white dark:hover:bg-gray-600',
+ )}
+ style={{ width: '100%' }}
+ data-theme={typeof item === 'string' ? item : (item as OptionType).value}
+ >
+ <span className="block truncate">
+ {typeof item === 'string' ? item : (item as OptionType).display}
+ </span>
+ </Listbox.Option>
+ ))}
+ </Listbox.Options>
+ </div>
+ </Listbox>
+ </div>
+ );
+};
+
+export default Dropdown;
diff --git a/client/src/hooks/Audio/useAudioRef.ts b/client/src/hooks/Audio/useAudioRef.ts
new file mode 100644
index 0000000000..b42b1e4794
--- /dev/null
+++ b/client/src/hooks/Audio/useAudioRef.ts
@@ -0,0 +1,48 @@
+import { useEffect, useRef } from 'react';
+
+export default function useCustomAudioRef({
+ setIsPlaying,
+}: {
+ setIsPlaying: (isPlaying: boolean) => void;
+}) {
+ const audioRef = useRef<HTMLAudioElement | null>(null);
+ useEffect(() => {
+ const handleEnded = () => {
+ setIsPlaying(false);
+ console.log('message audio ended');
+ if (audioRef.current) {
+ URL.revokeObjectURL(audioRef.current.src);
+ }
+ };
+
+ const handleStart = () => {
+ setIsPlaying(true);
+ console.log('message audio started');
+ };
+
+ const handlePause = () => {
+ setIsPlaying(false);
+ console.log('message audio paused');
+ };
+
+ const audioElement = audioRef.current;
+
+ if (audioRef.current) {
+ audioRef.current.muted = true;
+ audioRef.current.addEventListener('ended', handleEnded);
+ audioRef.current.addEventListener('play', handleStart);
+ audioRef.current.addEventListener('pause', handlePause);
+ }
+
+ return () => {
+ if (audioElement) {
+ audioElement.removeEventListener('ended', handleEnded);
+ audioElement.removeEventListener('play', handleStart);
+ audioElement.removeEventListener('pause', handlePause);
+ URL.revokeObjectURL(audioElement.src);
+ }
+ };
+ }, [setIsPlaying]);
+
+ return { audioRef };
+}
diff --git a/client/src/hooks/Input/useTextToSpeech.ts b/client/src/hooks/Input/useTextToSpeech.ts
index 40378d24ba..444248310a 100644
--- a/client/src/hooks/Input/useTextToSpeech.ts
+++ b/client/src/hooks/Input/useTextToSpeech.ts
@@ -1,13 +1,13 @@
import { useRef } from 'react';
import { parseTextParts } from 'librechat-data-provider';
-import type { TMessageContentParts } from 'librechat-data-provider';
+import type { TMessage } from 'librechat-data-provider';
import useTextToSpeechExternal from './useTextToSpeechExternal';
import useTextToSpeechBrowser from './useTextToSpeechBrowser';
import { usePauseGlobalAudio } from '../Audio';
import { useRecoilState } from 'recoil';
import store from '~/store';
-const useTextToSpeech = (message: string | TMessageContentParts[], isLast: boolean, index = 0) => {
+const useTextToSpeech = (message: TMessage, isLast: boolean, index = 0) => {
const [endpointTTS] = useRecoilState<string>(store.endpointTTS);
const useExternalTextToSpeech = endpointTTS === 'external';
@@ -22,7 +22,8 @@ const useTextToSpeech = (message: string | TMessageContentParts[], isLast: boole
cancelSpeech: cancelSpeechExternal,
isSpeaking: isSpeakingExternal,
isLoading: isLoading,
- } = useTextToSpeechExternal(isLast, index);
+ audioRef,
+ } = useTextToSpeechExternal(message.messageId, isLast, index);
const { pauseGlobalAudio } = usePauseGlobalAudio(index);
const generateSpeech = useExternalTextToSpeech ? generateSpeechExternal : generateSpeechLocal;
@@ -36,8 +37,10 @@ const useTextToSpeech = (message: string | TMessageContentParts[], isLast: boole
isMouseDownRef.current = true;
timerRef.current = window.setTimeout(() => {
if (isMouseDownRef.current) {
- const parsedMessage = typeof message === 'string' ? message : parseTextParts(message);
- generateSpeech(parsedMessage, true);
+ const messageContent = message?.content ?? message?.text ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
+ generateSpeech(parsedMessage, false);
}
}, 1000);
};
@@ -51,10 +54,13 @@ const useTextToSpeech = (message: string | TMessageContentParts[], isLast: boole
const toggleSpeech = () => {
if (isSpeaking) {
+ console.log('canceling message audio speech');
cancelSpeech();
pauseGlobalAudio();
} else {
- const parsedMessage = typeof message === 'string' ? message : parseTextParts(message);
+ const messageContent = message?.content ?? message?.text ?? '';
+ const parsedMessage =
+ typeof messageContent === 'string' ? messageContent : parseTextParts(messageContent);
generateSpeech(parsedMessage, false);
}
};
@@ -65,6 +71,7 @@ const useTextToSpeech = (message: string | TMessageContentParts[], isLast: boole
toggleSpeech,
isSpeaking,
isLoading,
+ audioRef,
};
};
diff --git a/client/src/hooks/Input/useTextToSpeechExternal.ts b/client/src/hooks/Input/useTextToSpeechExternal.ts
index 58dc82e6c4..2017cb737c 100644
--- a/client/src/hooks/Input/useTextToSpeechExternal.ts
+++ b/client/src/hooks/Input/useTextToSpeechExternal.ts
@@ -1,6 +1,7 @@
import { useRecoilValue } from 'recoil';
-import { useCallback, useEffect, useState, useMemo, useRef } from 'react';
+import { useState, useMemo, useRef, useCallback, useEffect } from 'react';
import { useTextToSpeechMutation } from '~/data-provider';
+import useAudioRef from '~/hooks/Audio/useAudioRef';
import useLocalize from '~/hooks/useLocalize';
import { useToastContext } from '~/Providers';
import store from '~/store';
@@ -12,23 +13,28 @@ const createFormData = (text: string, voice: string) => {
return formData;
};
-function useTextToSpeechExternal(isLast: boolean, index = 0) {
+function useTextToSpeechExternal(messageId: string, isLast: boolean, index = 0) {
const localize = useLocalize();
const { showToast } = useToastContext();
const voice = useRecoilValue(store.voice);
const cacheTTS = useRecoilValue(store.cacheTTS);
const playbackRate = useRecoilValue(store.playbackRate);
- const audioRef = useRef<HTMLAudioElement | null>(null);
-
const [downloadFile, setDownloadFile] = useState(false);
const [isLocalSpeaking, setIsSpeaking] = useState(false);
+ const { audioRef } = useAudioRef({ setIsPlaying: setIsSpeaking });
+ const promiseAudioRef = useRef<HTMLAudioElement | null>(null);
/* Global Audio Variables */
const globalIsFetching = useRecoilValue(store.globalAudioFetchingFamily(index));
const globalIsPlaying = useRecoilValue(store.globalAudioPlayingFamily(index));
- const playAudio = (blobUrl: string) => {
+ const autoPlayAudio = (blobUrl: string) => {
+ const newAudio = new Audio(blobUrl);
+ audioRef.current = newAudio;
+ };
+
+ const playAudioPromise = (blobUrl: string) => {
const newAudio = new Audio(blobUrl);
const initializeAudio = () => {
if (playbackRate && playbackRate !== 1) {
@@ -53,12 +59,12 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {
});
newAudio.onended = () => {
- console.log('Target message audio ended');
+ console.log('Cached message audio ended');
URL.revokeObjectURL(blobUrl);
setIsSpeaking(false);
};
- audioRef.current = newAudio;
+ promiseAudioRef.current = newAudio;
};
const downloadAudio = (blobUrl: string) => {
@@ -95,7 +101,7 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {
if (downloadFile) {
downloadAudio(blobUrl);
}
- playAudio(blobUrl);
+ autoPlayAudio(blobUrl);
} catch (error) {
showToast({
message: `Error processing audio: ${(error as Error).message}`,
@@ -111,38 +117,58 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {
},
});
- const generateSpeechExternal = async (text: string, download: boolean) => {
- const cachedResponse = await caches.match(text);
+ const startMutation = (text: string, download: boolean) => {
+ const formData = createFormData(text, voice);
+ setDownloadFile(download);
+ processAudio(formData);
+ };
- if (cachedResponse && cacheTTS) {
- handleCachedResponse(cachedResponse, download);
+ const generateSpeechExternal = (text: string, download: boolean) => {
+ if (cacheTTS) {
+ handleCachedResponse(text, download);
} else {
- const formData = createFormData(text, voice);
- setDownloadFile(download);
- processAudio(formData);
+ startMutation(text, download);
}
};
- const handleCachedResponse = async (cachedResponse: Response, download: boolean) => {
+ const handleCachedResponse = async (text: string, download: boolean) => {
+ const cachedResponse = await caches.match(text);
+ if (!cachedResponse) {
+ return startMutation(text, download);
+ }
const audioBlob = await cachedResponse.blob();
const blobUrl = URL.createObjectURL(audioBlob);
if (download) {
downloadAudio(blobUrl);
} else {
- playAudio(blobUrl);
+ playAudioPromise(blobUrl);
}
};
- const cancelSpeech = useCallback(() => {
- if (audioRef.current) {
- audioRef.current.pause();
- audioRef.current.src && URL.revokeObjectURL(audioRef.current.src);
- audioRef.current = null;
+ const cancelSpeech = () => {
+ const messageAudio = document.getElementById(`audio-${messageId}`) as HTMLAudioElement | null;
+ const pauseAudio = (currentElement: HTMLAudioElement | null) => {
+ if (currentElement) {
+ currentElement.pause();
+ currentElement.src && URL.revokeObjectURL(currentElement.src);
+ audioRef.current = null;
+ }
+ };
+ pauseAudio(messageAudio);
+ pauseAudio(promiseAudioRef.current);
+ setIsSpeaking(false);
+ };
+
+ const cancelPromiseSpeech = useCallback(() => {
+ if (promiseAudioRef.current) {
+ promiseAudioRef.current.pause();
+ promiseAudioRef.current.src && URL.revokeObjectURL(promiseAudioRef.current.src);
+ promiseAudioRef.current = null;
setIsSpeaking(false);
}
}, []);
- useEffect(() => cancelSpeech, [cancelSpeech]);
+ useEffect(() => cancelPromiseSpeech, [cancelPromiseSpeech]);
const isLoading = useMemo(() => {
return isProcessing || (isLast && globalIsFetching && !globalIsPlaying);
@@ -152,7 +178,7 @@ function useTextToSpeechExternal(isLast: boolean, index = 0) {
return isLocalSpeaking || (isLast && globalIsPlaying);
}, [isLocalSpeaking, globalIsPlaying, isLast]);
- return { generateSpeechExternal, cancelSpeech, isLoading, isSpeaking };
+ return { generateSpeechExternal, cancelSpeech, isLoading, isSpeaking, audioRef };
}
export default useTextToSpeechExternal;
diff --git a/client/src/hooks/Input/useTextarea.ts b/client/src/hooks/Input/useTextarea.ts
index 9e84d076e9..3b6c8b9cdd 100644
--- a/client/src/hooks/Input/useTextarea.ts
+++ b/client/src/hooks/Input/useTextarea.ts
@@ -10,6 +10,7 @@ import useGetSender from '~/hooks/Conversations/useGetSender';
import useFileHandling from '~/hooks/Files/useFileHandling';
import { useChatContext } from '~/Providers/ChatContext';
import useLocalize from '~/hooks/useLocalize';
+import { globalAudioId } from '~/common';
import store from '~/store';
type KeyEvent = KeyboardEvent<HTMLTextAreaElement>;
@@ -178,6 +179,11 @@ export default function useTextarea({
}
if (isNonShiftEnter && !isComposing?.current) {
+ const globalAudio = document.getElementById(globalAudioId) as HTMLAudioElement;
+ if (globalAudio) {
+ console.log('Unmuting global audio');
+ globalAudio.muted = false;
+ }
submitButtonRef.current?.click();
}
},