Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import { useEffect, useCallback } from 'react';
import { useParams } from 'react-router-dom';
import { useQueryClient } from '@tanstack/react-query';
import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';
import { QueryKeys, getSpeechText, getClientLimit } from 'librechat-data-provider';
import type { TMessage } from 'librechat-data-provider';
import {
useCustomAudioRef,
useAutoplayTrigger,
MediaSourceAppender,
usePauseGlobalAudio,
} from '~/hooks/Audio';
import {
logger,
getTTSCacheKey,
setCachedTTSAudio,
getCachedTTSAudio,
getTTSCacheGeneration,
} from '~/utils';
import { useAuthContext } from '~/hooks';
import { globalAudioId } from '~/common';
import store from '~/store';
function timeoutPromise(ms: number, message?: string) {
return new Promise((_, reject) =>
setTimeout(() => reject(new Error(message ?? 'Promise timed out')), ms),
);
}
const promiseTimeoutMessage = 'Reader promise timed out';
const maxPromiseTime = 15000;
export default function StreamAudio({ index = 0 }) {
const { token, user } = useAuthContext();
const cacheTTS = useRecoilValue(store.cacheTTS);
const playbackRate = useRecoilValue(store.playbackRate);
const voice = useRecoilValue(store.voice);
const automaticPlayback = useRecoilValue(store.automaticPlayback);
const setIsPlaying = useSetRecoilState(store.globalAudioPlayingFamily(index));
const setAudioRunId = useSetRecoilState(store.audioRunFamily(index));
const [isFetching, setIsFetching] = useRecoilState(store.globalAudioFetchingFamily(index));
const [globalAudioURL, setGlobalAudioURL] = useRecoilState(store.globalAudioURLFamily(index));
const { shouldPlay, activeRunId, latestMessage } = useAutoplayTrigger(index);
const { audioRef } = useCustomAudioRef({ setIsPlaying });
const { pauseGlobalAudio } = usePauseGlobalAudio();
const { conversationId: paramId } = useParams();
const queryParam = paramId === 'new' ? paramId : (latestMessage?.conversationId ?? paramId ?? '');
const queryClient = useQueryClient();
const getMessages = useCallback(
() => queryClient.getQueryData<TMessage[]>([QueryKeys.messages, queryParam]),
[queryParam, queryClient],
);
useEffect(() => {
if (!(token != null && automaticPlayback && shouldPlay && !isFetching)) {
return;
}
async function fetchAudio() {
setIsFetching(true);
const generation = getTTSCacheGeneration();
let mediaSource: MediaSourceAppender | undefined;
try {
if (audioRef.current) {
audioRef.current.pause();
URL.revokeObjectURL(audioRef.current.src);
setGlobalAudioURL(null);
}
/** Keyed by the spoken text, as manual playback is, so audio cached from an
* unfiltered message (reasoning included) is never replayed. */
let cacheKey = latestMessage
? await getTTSCacheKey(getSpeechText(latestMessage), voice, user?.id)
: null;
const cachedAudio = cacheKey ? getCachedTTSAudio(cacheKey) : undefined;
setAudioRunId(activeRunId);
if (cachedAudio) {
logger.log('Audio found in cache');
const blobUrl = URL.createObjectURL(cachedAudio);
setGlobalAudioURL(blobUrl);
setIsFetching(false);
return;
}
logger.log('Fetching audio...', navigator.userAgent);
const controller = new AbortController();
const abortTimer = setTimeout(() => controller.abort(), getClientLimit('ttsMs'));
let response: Response;
try {
response = await fetch('/api/files/speech/tts', {
method: 'POST',
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` },
body: JSON.stringify({
messageId: latestMessage?.messageId,
runId: activeRunId,
voice,
}),
signal: controller.signal,
});
} finally {
clearTimeout(abortTimer);
}
if (!response.ok) {
throw new Error('Failed to fetch audio');
}
if (!response.body) {
throw new Error('Null Response body');
}
const reader = response.body.getReader();
const type = 'audio/mpeg';
const browserSupportsType =
typeof MediaSource !== 'undefined' && MediaSource.isTypeSupported(type);
if (browserSupportsType) {
mediaSource = new MediaSourceAppender(type);
setGlobalAudioURL(mediaSource.mediaSourceUrl);
}
/** Browsers without MSE support for the type (Safari, for one) can only be handed the
* audio once it is fully read, so that path always buffers — caching opts MSE in too. */
const shouldBufferChunks = cacheTTS || !browserSupportsType;
let done = false;
const chunks: ArrayBuffer[] = [];
while (!done) {
const readPromise = reader.read();
const { value, done: readerDone } = (await Promise.race([
readPromise,
timeoutPromise(maxPromiseTime, promiseTimeoutMessage),
])) as ReadableStreamReadResult<ArrayBuffer>;
if (shouldBufferChunks && value) {
chunks.push(value);
}
if (value && mediaSource) {
mediaSource.addData(value);
}
done = readerDone;
}
mediaSource?.close();
if (chunks.length) {
const audioBlob = new Blob(chunks, { type });
if (!browserSupportsType) {
setGlobalAudioURL(URL.createObjectURL(audioBlob));
}
if (cacheTTS) {
const latestMessages = getMessages() ?? [];
const targetMessage = latestMessages.find(
(msg) => msg.messageId === latestMessage?.messageId,
);
cacheKey = targetMessage
? await getTTSCacheKey(getSpeechText(targetMessage), voice, user?.id)
: null;
if (!cacheKey) {
logger.warn('Cache key not found, skipping audio cache');
} else {
logger.log('Adding audio to cache');
setCachedTTSAudio(cacheKey, audioBlob, generation);
}
}
}
logger.log('Audio stream reading ended');
} catch (error) {
if (error?.['message'] === promiseTimeoutMessage) {
logger.log(promiseTimeoutMessage);
/** Let whatever was already appended finish playing instead of stalling forever */
mediaSource?.close();
return;
}
logger.error('Error fetching audio:', error);
setGlobalAudioURL(null);
} finally {
setIsFetching(false);
}
}
fetchAudio();
}, [
automaticPlayback,
setGlobalAudioURL,
setAudioRunId,
setIsFetching,
latestMessage,
activeRunId,
getMessages,
shouldPlay,
isFetching,
cacheTTS,
audioRef,
voice,
token,
user?.id,
]);
useEffect(() => {
if (
playbackRate != null &&
globalAudioURL != null &&
playbackRate > 0 &&
audioRef.current &&
audioRef.current.playbackRate !== playbackRate
) {
audioRef.current.playbackRate = playbackRate;
}
}, [audioRef, globalAudioURL, playbackRate]);
useEffect(() => {
pauseGlobalAudio();
// We only want the effect to run when the paramId changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [paramId]);
logger.log('StreamAudio.tsx - globalAudioURL:', globalAudioURL);
return (
// eslint-disable-next-line jsx-a11y/media-has-caption
<audio
ref={audioRef}
controls
controlsList="nodownload nofullscreen noremoteplayback"
className="absolute hidden size-0 overflow-hidden"
src={globalAudioURL ?? undefined}
id={globalAudioId}
autoPlay
/>
);
}