Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🗣️ refactor: speech services; fix: OpenAI STT (#3431)

Marco Beretta51cd847606eb
diff --git a/api/server/services/Files/Audio/STTService.js b/api/server/services/Files/Audio/STTService.js
new file mode 100644
index 0000000000..49de1c84d6
--- /dev/null
+++ b/api/server/services/Files/Audio/STTService.js
@@ -0,0 +1,248 @@
+const { Readable } = require('stream');
+const axios = require('axios');
+const { extractEnvVariable, STTProviders } = require('librechat-data-provider');
+const getCustomConfig = require('~/server/services/Config/getCustomConfig');
+const { genAzureEndpoint } = require('~/utils');
+const { logger } = require('~/config');
+
+/**
+ * Service class for handling Speech-to-Text (STT) operations.
+ * @class
+ */
+class STTService {
+ /**
+ * Creates an instance of STTService.
+ * @param {Object} customConfig - The custom configuration object.
+ */
+ constructor(customConfig) {
+ this.customConfig = customConfig;
+ this.providerStrategies = {
+ [STTProviders.OPENAI]: this.openAIProvider,
+ [STTProviders.AZURE_OPENAI]: this.azureOpenAIProvider,
+ };
+ }
+
+ /**
+ * Creates a singleton instance of STTService.
+ * @static
+ * @async
+ * @returns {Promise<STTService>} The STTService instance.
+ * @throws {Error} If the custom config is not found.
+ */
+ static async getInstance() {
+ const customConfig = await getCustomConfig();
+ if (!customConfig) {
+ throw new Error('Custom config not found');
+ }
+ return new STTService(customConfig);
+ }
+
+ /**
+ * Retrieves the configured STT provider and its schema.
+ * @returns {Promise<[string, Object]>} A promise that resolves to an array containing the provider name and its schema.
+ * @throws {Error} If no STT schema is set, multiple providers are set, or no provider is set.
+ */
+ async getProviderSchema() {
+ const sttSchema = this.customConfig.speech.stt;
+
+ if (!sttSchema) {
+ throw new Error(
+ 'No STT schema is set. Did you configure STT in the custom config (librechat.yaml)?',
+ );
+ }
+
+ const providers = Object.entries(sttSchema).filter(
+ ([, value]) => Object.keys(value).length > 0,
+ );
+
+ if (providers.length !== 1) {
+ throw new Error(
+ providers.length > 1
+ ? 'Multiple providers are set. Please set only one provider.'
+ : 'No provider is set. Please set a provider.',
+ );
+ }
+
+ const [provider, schema] = providers[0];
+ return [provider, schema];
+ }
+
+ /**
+ * Recursively removes undefined properties from an object.
+ * @param {Object} obj - The object to clean.
+ * @returns {void}
+ */
+ removeUndefined(obj) {
+ Object.keys(obj).forEach((key) => {
+ if (obj[key] && typeof obj[key] === 'object') {
+ this.removeUndefined(obj[key]);
+ if (Object.keys(obj[key]).length === 0) {
+ delete obj[key];
+ }
+ } else if (obj[key] === undefined) {
+ delete obj[key];
+ }
+ });
+ }
+
+ /**
+ * Prepares the request for the OpenAI STT provider.
+ * @param {Object} sttSchema - The STT schema for OpenAI.
+ * @param {Stream} audioReadStream - The audio data to be transcribed.
+ * @returns {Array} An array containing the URL, data, and headers for the request.
+ */
+ openAIProvider(sttSchema, audioReadStream) {
+ const url = sttSchema?.url || 'https://api.openai.com/v1/audio/transcriptions';
+ const apiKey = extractEnvVariable(sttSchema.apiKey) || '';
+
+ const data = {
+ file: audioReadStream,
+ model: sttSchema.model,
+ };
+
+ const headers = {
+ 'Content-Type': 'multipart/form-data',
+ ...(apiKey && { Authorization: `Bearer ${apiKey}` }),
+ };
+ [headers].forEach(this.removeUndefined);
+
+ return [url, data, headers];
+ }
+
+ /**
+ * Prepares the request for the Azure OpenAI STT provider.
+ * @param {Object} sttSchema - The STT schema for Azure OpenAI.
+ * @param {Buffer} audioBuffer - The audio data to be transcribed.
+ * @param {Object} audioFile - The audio file object containing originalname, mimetype, and size.
+ * @returns {Array} An array containing the URL, data, and headers for the request.
+ * @throws {Error} If the audio file size exceeds 25MB or the audio file format is not accepted.
+ */
+ azureOpenAIProvider(sttSchema, audioBuffer, audioFile) {
+ const url = `${genAzureEndpoint({
+ azureOpenAIApiInstanceName: sttSchema?.instanceName,
+ azureOpenAIApiDeploymentName: sttSchema?.deploymentName,
+ })}/audio/transcriptions?api-version=${sttSchema?.apiVersion}`;
+
+ const apiKey = sttSchema.apiKey ? extractEnvVariable(sttSchema.apiKey) : '';
+
+ if (audioBuffer.byteLength > 25 * 1024 * 1024) {
+ throw new Error('The audio file size exceeds the limit of 25MB');
+ }
+
+ const acceptedFormats = ['flac', 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'ogg', 'wav', 'webm'];
+ const fileFormat = audioFile.mimetype.split('/')[1];
+ if (!acceptedFormats.includes(fileFormat)) {
+ throw new Error(`The audio file format ${fileFormat} is not accepted`);
+ }
+
+ const formData = new FormData();
+ const audioBlob = new Blob([audioBuffer], { type: audioFile.mimetype });
+ formData.append('file', audioBlob, audioFile.originalname);
+
+ const headers = {
+ 'Content-Type': 'multipart/form-data',
+ ...(apiKey && { 'api-key': apiKey }),
+ };
+
+ [headers].forEach(this.removeUndefined);
+
+ return [url, formData, headers];
+ }
+
+ /**
+ * Sends an STT request to the specified provider.
+ * @async
+ * @param {string} provider - The STT provider to use.
+ * @param {Object} sttSchema - The STT schema for the provider.
+ * @param {Object} requestData - The data required for the STT request.
+ * @param {Buffer} requestData.audioBuffer - The audio data to be transcribed.
+ * @param {Object} requestData.audioFile - The audio file object containing originalname, mimetype, and size.
+ * @returns {Promise<string>} A promise that resolves to the transcribed text.
+ * @throws {Error} If the provider is invalid, the response status is not 200, or the response data is missing.
+ */
+ async sttRequest(provider, sttSchema, { audioBuffer, audioFile }) {
+ const strategy = this.providerStrategies[provider];
+ if (!strategy) {
+ throw new Error('Invalid provider');
+ }
+
+ const audioReadStream = Readable.from(audioBuffer);
+ audioReadStream.path = 'audio.wav';
+
+ const [url, data, headers] = strategy.call(this, sttSchema, audioReadStream, audioFile);
+
+ if (!Readable.from && data instanceof FormData) {
+ const audioBlob = new Blob([audioBuffer], { type: audioFile.mimetype });
+ data.set('file', audioBlob, audioFile.originalname);
+ }
+
+ try {
+ const response = await axios.post(url, data, { headers });
+
+ if (response.status !== 200) {
+ throw new Error('Invalid response from the STT API');
+ }
+
+ if (!response.data || !response.data.text) {
+ throw new Error('Missing data in response from the STT API');
+ }
+
+ return response.data.text.trim();
+ } catch (error) {
+ logger.error(`STT request failed for provider ${provider}:`, error);
+ throw error;
+ }
+ }
+
+ /**
+ * Processes a speech-to-text request.
+ * @async
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @returns {Promise<void>}
+ */
+ async processTextToSpeech(req, res) {
+ if (!req.file || !req.file.buffer) {
+ return res.status(400).json({ message: 'No audio file provided in the FormData' });
+ }
+
+ const audioBuffer = req.file.buffer;
+ const audioFile = {
+ originalname: req.file.originalname,
+ mimetype: req.file.mimetype,
+ size: req.file.size,
+ };
+
+ try {
+ const [provider, sttSchema] = await this.getProviderSchema();
+ const text = await this.sttRequest(provider, sttSchema, { audioBuffer, audioFile });
+ res.json({ text });
+ } catch (error) {
+ logger.error('An error occurred while processing the audio:', error);
+ res.sendStatus(500);
+ }
+ }
+}
+
+/**
+ * Factory function to create an STTService instance.
+ * @async
+ * @returns {Promise<STTService>} A promise that resolves to an STTService instance.
+ */
+async function createSTTService() {
+ return STTService.getInstance();
+}
+
+/**
+ * Wrapper function for speech-to-text processing.
+ * @async
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @returns {Promise<void>}
+ */
+async function speechToText(req, res) {
+ const sttService = await createSTTService();
+ await sttService.processTextToSpeech(req, res);
+}
+
+module.exports = speechToText;
diff --git a/api/server/services/Files/Audio/TTSService.js b/api/server/services/Files/Audio/TTSService.js
new file mode 100644
index 0000000000..cf99633cb7
--- /dev/null
+++ b/api/server/services/Files/Audio/TTSService.js
@@ -0,0 +1,477 @@
+const axios = require('axios');
+const { extractEnvVariable, TTSProviders } = require('librechat-data-provider');
+const { logger } = require('~/config');
+const getCustomConfig = require('~/server/services/Config/getCustomConfig');
+const { genAzureEndpoint } = require('~/utils');
+const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio');
+
+/**
+ * Service class for handling Text-to-Speech (TTS) operations.
+ * @class
+ */
+class TTSService {
+ /**
+ * Creates an instance of TTSService.
+ * @param {Object} customConfig - The custom configuration object.
+ */
+ constructor(customConfig) {
+ this.customConfig = customConfig;
+ this.providerStrategies = {
+ [TTSProviders.OPENAI]: this.openAIProvider.bind(this),
+ [TTSProviders.AZURE_OPENAI]: this.azureOpenAIProvider.bind(this),
+ [TTSProviders.ELEVENLABS]: this.elevenLabsProvider.bind(this),
+ [TTSProviders.LOCALAI]: this.localAIProvider.bind(this),
+ };
+ }
+
+ /**
+ * Creates a singleton instance of TTSService.
+ * @static
+ * @async
+ * @returns {Promise<TTSService>} The TTSService instance.
+ * @throws {Error} If the custom config is not found.
+ */
+ static async getInstance() {
+ const customConfig = await getCustomConfig();
+ if (!customConfig) {
+ throw new Error('Custom config not found');
+ }
+ return new TTSService(customConfig);
+ }
+
+ /**
+ * Retrieves the configured TTS provider.
+ * @returns {string} The name of the configured provider.
+ * @throws {Error} If no provider is set or multiple providers are set.
+ */
+ getProvider() {
+ const ttsSchema = this.customConfig.speech.tts;
+ if (!ttsSchema) {
+ throw new Error(
+ 'No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)?',
+ );
+ }
+ const providers = Object.entries(ttsSchema).filter(
+ ([, value]) => Object.keys(value).length > 0,
+ );
+
+ if (providers.length !== 1) {
+ throw new Error(
+ providers.length > 1
+ ? 'Multiple providers are set. Please set only one provider.'
+ : 'No provider is set. Please set a provider.',
+ );
+ }
+ return providers[0][0];
+ }
+
+ /**
+ * Selects a voice for TTS based on provider schema and request.
+ * @async
+ * @param {Object} providerSchema - The schema for the selected provider.
+ * @param {string} requestVoice - The requested voice.
+ * @returns {Promise<string>} The selected voice.
+ */
+ async getVoice(providerSchema, requestVoice) {
+ const voices = providerSchema.voices.filter((voice) => voice && voice.toUpperCase() !== 'ALL');
+ let voice = requestVoice;
+ if (!voice || !voices.includes(voice) || (voice.toUpperCase() === 'ALL' && voices.length > 1)) {
+ voice = getRandomVoiceId(voices);
+ }
+ return voice;
+ }
+
+ /**
+ * Recursively removes undefined properties from an object.
+ * @param {Object} obj - The object to clean.
+ */
+ removeUndefined(obj) {
+ Object.keys(obj).forEach((key) => {
+ if (obj[key] && typeof obj[key] === 'object') {
+ this.removeUndefined(obj[key]);
+ if (Object.keys(obj[key]).length === 0) {
+ delete obj[key];
+ }
+ } else if (obj[key] === undefined) {
+ delete obj[key];
+ }
+ });
+ }
+
+ /**
+ * Prepares the request for OpenAI TTS provider.
+ * @param {Object} ttsSchema - The TTS schema for OpenAI.
+ * @param {string} input - The input text.
+ * @param {string} voice - The selected voice.
+ * @returns {Array} An array containing the URL, data, and headers for the request.
+ * @throws {Error} If the selected voice is not available.
+ */
+ openAIProvider(ttsSchema, input, voice) {
+ const url = ttsSchema?.url || 'https://api.openai.com/v1/audio/speech';
+
+ if (
+ ttsSchema?.voices &&
+ ttsSchema.voices.length > 0 &&
+ !ttsSchema.voices.includes(voice) &&
+ !ttsSchema.voices.includes('ALL')
+ ) {
+ throw new Error(`Voice ${voice} is not available.`);
+ }
+
+ const data = {
+ input,
+ model: ttsSchema?.model,
+ voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,
+ backend: ttsSchema?.backend,
+ };
+
+ const headers = {
+ 'Content-Type': 'application/json',
+ Authorization: `Bearer ${extractEnvVariable(ttsSchema?.apiKey)}`,
+ };
+
+ return [url, data, headers];
+ }
+
+ /**
+ * Prepares the request for Azure OpenAI TTS provider.
+ * @param {Object} ttsSchema - The TTS schema for Azure OpenAI.
+ * @param {string} input - The input text.
+ * @param {string} voice - The selected voice.
+ * @returns {Array} An array containing the URL, data, and headers for the request.
+ * @throws {Error} If the selected voice is not available.
+ */
+ azureOpenAIProvider(ttsSchema, input, voice) {
+ const url = `${genAzureEndpoint({
+ azureOpenAIApiInstanceName: ttsSchema?.instanceName,
+ azureOpenAIApiDeploymentName: ttsSchema?.deploymentName,
+ })}/audio/speech?api-version=${ttsSchema?.apiVersion}`;
+
+ if (
+ ttsSchema?.voices &&
+ ttsSchema.voices.length > 0 &&
+ !ttsSchema.voices.includes(voice) &&
+ !ttsSchema.voices.includes('ALL')
+ ) {
+ throw new Error(`Voice ${voice} is not available.`);
+ }
+
+ const data = {
+ model: ttsSchema?.model,
+ input,
+ voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,
+ };
+
+ const headers = {
+ 'Content-Type': 'application/json',
+ 'api-key': ttsSchema.apiKey ? extractEnvVariable(ttsSchema.apiKey) : '',
+ };
+
+ return [url, data, headers];
+ }
+
+ /**
+ * Prepares the request for ElevenLabs TTS provider.
+ * @param {Object} ttsSchema - The TTS schema for ElevenLabs.
+ * @param {string} input - The input text.
+ * @param {string} voice - The selected voice.
+ * @param {boolean} stream - Whether to use streaming.
+ * @returns {Array} An array containing the URL, data, and headers for the request.
+ * @throws {Error} If the selected voice is not available.
+ */
+ elevenLabsProvider(ttsSchema, input, voice, stream) {
+ let url =
+ ttsSchema?.url ||
+ `https://api.elevenlabs.io/v1/text-to-speech/${voice}${stream ? '/stream' : ''}`;
+
+ if (!ttsSchema?.voices.includes(voice) && !ttsSchema?.voices.includes('ALL')) {
+ throw new Error(`Voice ${voice} is not available.`);
+ }
+
+ const data = {
+ model_id: ttsSchema?.model,
+ text: input,
+ voice_settings: {
+ similarity_boost: ttsSchema?.voice_settings?.similarity_boost,
+ stability: ttsSchema?.voice_settings?.stability,
+ style: ttsSchema?.voice_settings?.style,
+ use_speaker_boost: ttsSchema?.voice_settings?.use_speaker_boost,
+ },
+ pronunciation_dictionary_locators: ttsSchema?.pronunciation_dictionary_locators,
+ };
+
+ const headers = {
+ 'Content-Type': 'application/json',
+ 'xi-api-key': extractEnvVariable(ttsSchema?.apiKey),
+ Accept: 'audio/mpeg',
+ };
+
+ return [url, data, headers];
+ }
+
+ /**
+ * Prepares the request for LocalAI TTS provider.
+ * @param {Object} ttsSchema - The TTS schema for LocalAI.
+ * @param {string} input - The input text.
+ * @param {string} voice - The selected voice.
+ * @returns {Array} An array containing the URL, data, and headers for the request.
+ * @throws {Error} If the selected voice is not available.
+ */
+ localAIProvider(ttsSchema, input, voice) {
+ const url = ttsSchema?.url;
+
+ if (
+ ttsSchema?.voices &&
+ ttsSchema.voices.length > 0 &&
+ !ttsSchema.voices.includes(voice) &&
+ !ttsSchema.voices.includes('ALL')
+ ) {
+ throw new Error(`Voice ${voice} is not available.`);
+ }
+
+ const data = {
+ input,
+ model: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,
+ backend: ttsSchema?.backend,
+ };
+
+ const headers = {
+ 'Content-Type': 'application/json',
+ Authorization: `Bearer ${extractEnvVariable(ttsSchema?.apiKey)}`,
+ };
+
+ if (extractEnvVariable(ttsSchema.apiKey) === '') {
+ delete headers.Authorization;
+ }
+
+ return [url, data, headers];
+ }
+
+ /**
+ * Sends a TTS request to the specified provider.
+ * @async
+ * @param {string} provider - The TTS provider to use.
+ * @param {Object} ttsSchema - The TTS schema for the provider.
+ * @param {Object} options - The options for the TTS request.
+ * @param {string} options.input - The input text.
+ * @param {string} options.voice - The voice to use.
+ * @param {boolean} [options.stream=true] - Whether to use streaming.
+ * @returns {Promise<Object>} The axios response object.
+ * @throws {Error} If the provider is invalid or the request fails.
+ */
+ async ttsRequest(provider, ttsSchema, { input, voice, stream = true }) {
+ const strategy = this.providerStrategies[provider];
+ if (!strategy) {
+ throw new Error('Invalid provider');
+ }
+
+ const [url, data, headers] = strategy.call(this, ttsSchema, input, voice, stream);
+
+ [data, headers].forEach(this.removeUndefined.bind(this));
+
+ const options = { headers, responseType: stream ? 'stream' : 'arraybuffer' };
+
+ try {
+ return await axios.post(url, data, options);
+ } catch (error) {
+ logger.error(`TTS request failed for provider ${provider}:`, error);
+ throw error;
+ }
+ }
+
+ /**
+ * Processes a text-to-speech request.
+ * @async
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @returns {Promise<void>}
+ */
+ async processTextToSpeech(req, res) {
+ const { input, voice: requestVoice } = req.body;
+
+ if (!input) {
+ return res.status(400).send('Missing text in request body');
+ }
+
+ try {
+ res.setHeader('Content-Type', 'audio/mpeg');
+ const provider = this.getProvider();
+ const ttsSchema = this.customConfig.speech.tts[provider];
+ const voice = await this.getVoice(ttsSchema, requestVoice);
+
+ if (input.length < 4096) {
+ const response = await this.ttsRequest(provider, ttsSchema, { input, voice });
+ response.data.pipe(res);
+ return;
+ }
+
+ const textChunks = splitTextIntoChunks(input, 1000);
+
+ for (const chunk of textChunks) {
+ try {
+ const response = await this.ttsRequest(provider, ttsSchema, {
+ voice,
+ input: chunk.text,
+ stream: true,
+ });
+
+ logger.debug(`[textToSpeech] user: ${req?.user?.id} | writing audio stream`);
+ await new Promise((resolve) => {
+ response.data.pipe(res, { end: chunk.isFinished });
+ response.data.on('end', resolve);
+ });
+
+ if (chunk.isFinished) {
+ break;
+ }
+ } catch (innerError) {
+ logger.error('Error processing manual update:', chunk, innerError);
+ if (!res.headersSent) {
+ return res.status(500).end();
+ }
+ return;
+ }
+ }
+
+ if (!res.headersSent) {
+ res.end();
+ }
+ } catch (error) {
+ logger.error('Error creating the audio stream:', error);
+ if (!res.headersSent) {
+ return res.status(500).send('An error occurred');
+ }
+ }
+ }
+
+ /**
+ * Streams audio data from the TTS provider.
+ * @async
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @returns {Promise<void>}
+ */
+ async streamAudio(req, res) {
+ res.setHeader('Content-Type', 'audio/mpeg');
+ const provider = this.getProvider();
+ const ttsSchema = this.customConfig.speech.tts[provider];
+ const voice = await this.getVoice(ttsSchema, req.body.voice);
+
+ let shouldContinue = true;
+
+ req.on('close', () => {
+ logger.warn('[streamAudio] Audio Stream Request closed by client');
+ shouldContinue = false;
+ });
+
+ const processChunks = createChunkProcessor(req.body.messageId);
+
+ try {
+ while (shouldContinue) {
+ const updates = await processChunks();
+ if (typeof updates === 'string') {
+ logger.error(`Error processing audio stream updates: ${updates}`);
+ return res.status(500).end();
+ }
+
+ if (updates.length === 0) {
+ await new Promise((resolve) => setTimeout(resolve, 1250));
+ continue;
+ }
+
+ for (const update of updates) {
+ try {
+ const response = await this.ttsRequest(provider, ttsSchema, {
+ voice,
+ input: update.text,
+ stream: true,
+ });
+
+ if (!shouldContinue) {
+ break;
+ }
+
+ logger.debug(`[streamAudio] user: ${req?.user?.id} | writing audio stream`);
+ await new Promise((resolve) => {
+ response.data.pipe(res, { end: update.isFinished });
+ response.data.on('end', resolve);
+ });
+
+ if (update.isFinished) {
+ shouldContinue = false;
+ break;
+ }
+ } catch (innerError) {
+ logger.error('Error processing audio stream update:', update, innerError);
+ if (!res.headersSent) {
+ return res.status(500).end();
+ }
+ return;
+ }
+ }
+
+ if (!shouldContinue) {
+ break;
+ }
+ }
+
+ if (!res.headersSent) {
+ res.end();
+ }
+ } catch (error) {
+ logger.error('Failed to fetch audio:', error);
+ if (!res.headersSent) {
+ res.status(500).end();
+ }
+ }
+ }
+}
+
+/**
+ * Factory function to create a TTSService instance.
+ * @async
+ * @returns {Promise<TTSService>} A promise that resolves to a TTSService instance.
+ */
+async function createTTSService() {
+ return TTSService.getInstance();
+}
+
+/**
+ * Wrapper function for text-to-speech processing.
+ * @async
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @returns {Promise<void>}
+ */
+async function textToSpeech(req, res) {
+ const ttsService = await createTTSService();
+ await ttsService.processTextToSpeech(req, res);
+}
+
+/**
+ * Wrapper function for audio streaming.
+ * @async
+ * @param {Object} req - The request object.
+ * @param {Object} res - The response object.
+ * @returns {Promise<void>}
+ */
+async function streamAudio(req, res) {
+ const ttsService = await createTTSService();
+ await ttsService.streamAudio(req, res);
+}
+
+/**
+ * Wrapper function to get the configured TTS provider.
+ * @async
+ * @returns {Promise<string>} A promise that resolves to the name of the configured provider.
+ */
+async function getProvider() {
+ const ttsService = await createTTSService();
+ return ttsService.getProvider();
+}
+
+module.exports = {
+ textToSpeech,
+ streamAudio,
+ getProvider,
+};
diff --git a/api/server/services/Files/Audio/getVoices.js b/api/server/services/Files/Audio/getVoices.js
index 6a543bb4a4..3a0618b772 100644
--- a/api/server/services/Files/Audio/getVoices.js
+++ b/api/server/services/Files/Audio/getVoices.js
@@ -1,6 +1,6 @@
const { TTSProviders } = require('librechat-data-provider');
const getCustomConfig = require('~/server/services/Config/getCustomConfig');
-const { getProvider } = require('./textToSpeech');
+const { getProvider } = require('./TTSService');
/**
* This function retrieves the available voices for the current TTS provider
diff --git a/api/server/services/Files/Audio/index.js b/api/server/services/Files/Audio/index.js
index 75882f2397..b698e9da87 100644
--- a/api/server/services/Files/Audio/index.js
+++ b/api/server/services/Files/Audio/index.js
@@ -1,11 +1,11 @@
const getVoices = require('./getVoices');
const getCustomConfigSpeech = require('./getCustomConfigSpeech');
-const textToSpeech = require('./textToSpeech');
-const speechToText = require('./speechToText');
+const TTSService = require('./TTSService');
+const STTService = require('./STTService');
module.exports = {
getVoices,
getCustomConfigSpeech,
- speechToText,
- ...textToSpeech,
+ ...STTService,
+ ...TTSService,
};
diff --git a/api/server/services/Files/Audio/speechToText.js b/api/server/services/Files/Audio/speechToText.js
deleted file mode 100644
index f6b52f8b7c..0000000000
--- a/api/server/services/Files/Audio/speechToText.js
+++ /dev/null
@@ -1,225 +0,0 @@
-const { Readable } = require('stream');
-const axios = require('axios');
-const { extractEnvVariable, STTProviders } = require('librechat-data-provider');
-const getCustomConfig = require('~/server/services/Config/getCustomConfig');
-const { genAzureEndpoint } = require('~/utils');
-const { logger } = require('~/config');
-
-/**
- * Handle the response from the STT API
- * @param {Object} response - The response from the STT API
- *
- * @returns {string} The text from the response data
- *
- * @throws Will throw an error if the response status is not 200 or the response data is missing
- */
-async function handleResponse(response) {
- if (response.status !== 200) {
- throw new Error('Invalid response from the STT API');
- }
-
- if (!response.data || !response.data.text) {
- throw new Error('Missing data in response from the STT API');
- }
-
- return response.data.text.trim();
-}
-
-/**
- * getProviderSchema function
- * This function takes the customConfig object and returns the name of the provider and its schema
- * If more than one provider is set or no provider is set, it throws an error
- *
- * @param {Object} customConfig - The custom configuration containing the STT schema
- * @returns {Promise<[string, Object]>} The name of the provider and its schema
- * @throws {Error} Throws an error if multiple providers are set or no provider is set
- */
-async function getProviderSchema(customConfig) {
- const sttSchema = customConfig.speech.stt;
-
- if (!sttSchema) {
- throw new Error(`No STT schema is set. Did you configure STT in the custom config (librechat.yaml)?
-
- https://www.librechat.ai/docs/configuration/stt_tts#stt`);
- }
-
- const providers = Object.entries(sttSchema).filter(([, value]) => Object.keys(value).length > 0);
-
- if (providers.length > 1) {
- throw new Error('Multiple providers are set. Please set only one provider.');
- } else if (providers.length === 0) {
- throw new Error('No provider is set. Please set a provider.');
- } else {
- const provider = providers[0][0];
- return [provider, sttSchema[provider]];
- }
-}
-
-function removeUndefined(obj) {
- Object.keys(obj).forEach((key) => {
- if (obj[key] && typeof obj[key] === 'object') {
- removeUndefined(obj[key]);
- if (Object.keys(obj[key]).length === 0) {
- delete obj[key];
- }
- } else if (obj[key] === undefined) {
- delete obj[key];
- }
- });
-}
-
-/**
- * This function prepares the necessary data and headers for making a request to the OpenAI API
- * It uses the provided speech-to-text schema and audio stream to create the request
- *
- * @param {Object} sttSchema - The speech-to-text schema containing the OpenAI configuration
- * @param {Stream} audioReadStream - The audio data to be transcribed
- *
- * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
- * If an error occurs, it returns an array with three null values and logs the error with logger
- */
-function openAIProvider(sttSchema, audioReadStream) {
- try {
- const url = sttSchema.openai?.url || 'https://api.openai.com/v1/audio/transcriptions';
- const apiKey = sttSchema.openai.apiKey ? extractEnvVariable(sttSchema.openai.apiKey) : '';
-
- let data = {
- file: audioReadStream,
- model: sttSchema.openai.model,
- };
-
- let headers = {
- 'Content-Type': 'multipart/form-data',
- };
-
- [headers].forEach(removeUndefined);
-
- if (apiKey) {
- headers.Authorization = 'Bearer ' + apiKey;
- }
-
- return [url, data, headers];
- } catch (error) {
- logger.error('An error occurred while preparing the OpenAI API STT request: ', error);
- return [null, null, null];
- }
-}
-
-/**
- * Prepares the necessary data and headers for making a request to the Azure API.
- * It uses the provided Speech-to-Text (STT) schema and audio file to create the request.
- *
- * @param {Object} sttSchema - The STT schema object, which should contain instanceName, deploymentName, apiVersion, and apiKey.
- * @param {Buffer} audioBuffer - The audio data to be transcribed
- * @param {Object} audioFile - The audio file object, which should contain originalname, mimetype, and size.
- *
- * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request.
- * If an error occurs, it logs the error with logger and returns an array with three null values.
- */
-function azureOpenAIProvider(sttSchema, audioBuffer, audioFile) {
- try {
- const instanceName = sttSchema?.instanceName;
- const deploymentName = sttSchema?.deploymentName;
- const apiVersion = sttSchema?.apiVersion;
-
- const url =
- genAzureEndpoint({
- azureOpenAIApiInstanceName: instanceName,
- azureOpenAIApiDeploymentName: deploymentName,
- }) +
- '/audio/transcriptions?api-version=' +
- apiVersion;
-
- const apiKey = sttSchema.apiKey ? extractEnvVariable(sttSchema.apiKey) : '';
-
- if (audioBuffer.byteLength > 25 * 1024 * 1024) {
- throw new Error('The audio file size exceeds the limit of 25MB');
- }
- const acceptedFormats = ['flac', 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'ogg', 'wav', 'webm'];
- const fileFormat = audioFile.mimetype.split('/')[1];
- if (!acceptedFormats.includes(fileFormat)) {
- throw new Error(`The audio file format ${fileFormat} is not accepted`);
- }
-
- const formData = new FormData();
-
- const audioBlob = new Blob([audioBuffer], { type: audioFile.mimetype });
-
- formData.append('file', audioBlob, audioFile.originalname);
-
- let data = formData;
-
- let headers = {
- 'Content-Type': 'multipart/form-data',
- };
-
- [headers].forEach(removeUndefined);
-
- if (apiKey) {
- headers['api-key'] = apiKey;
- }
-
- return [url, data, headers];
- } catch (error) {
- logger.error('An error occurred while preparing the Azure OpenAI API STT request: ', error);
- throw error;
- }
-}
-
-/**
- * Convert speech to text
- * @param {Object} req - The request object
- * @param {Object} res - The response object
- *
- * @returns {Object} The response object with the text from the STT API
- *
- * @throws Will throw an error if an error occurs while processing the audio
- */
-
-async function speechToText(req, res) {
- const customConfig = await getCustomConfig();
- if (!customConfig) {
- return res.status(500).send('Custom config not found');
- }
-
- if (!req.file || !req.file.buffer) {
- return res.status(400).json({ message: 'No audio file provided in the FormData' });
- }
-
- const audioBuffer = req.file.buffer;
- const audioReadStream = Readable.from(audioBuffer);
- audioReadStream.path = 'audio.wav';
-
- const [provider, sttSchema] = await getProviderSchema(customConfig);
-
- let [url, data, headers] = [];
-
- switch (provider) {
- case STTProviders.OPENAI:
- [url, data, headers] = openAIProvider(sttSchema, audioReadStream);
- break;
- case STTProviders.AZURE_OPENAI:
- [url, data, headers] = azureOpenAIProvider(sttSchema, audioBuffer, req.file);
- break;
- default:
- throw new Error('Invalid provider');
- }
-
- if (!Readable.from) {
- const audioBlob = new Blob([audioBuffer], { type: req.file.mimetype });
- delete data['file'];
- data['file'] = audioBlob;
- }
-
- try {
- const response = await axios.post(url, data, { headers: headers });
- const text = await handleResponse(response);
-
- res.json({ text });
- } catch (error) {
- logger.error('An error occurred while processing the audio:', error);
- res.sendStatus(500);
- }
-}
-
-module.exports = speechToText;
diff --git a/api/server/services/Files/Audio/textToSpeech.js b/api/server/services/Files/Audio/textToSpeech.js
deleted file mode 100644
index 2d04a75236..0000000000
--- a/api/server/services/Files/Audio/textToSpeech.js
+++ /dev/null
@@ -1,473 +0,0 @@
-const axios = require('axios');
-const { extractEnvVariable, TTSProviders } = require('librechat-data-provider');
-const { logger } = require('~/config');
-const getCustomConfig = require('~/server/services/Config/getCustomConfig');
-const { genAzureEndpoint } = require('~/utils');
-const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio');
-
-/**
- * getProvider function
- * This function takes the ttsSchema object and returns the name of the provider
- * If more than one provider is set or no provider is set, it throws an error
- *
- * @param {Object} ttsSchema - The TTS schema containing the provider configuration
- * @returns {string} The name of the provider
- * @throws {Error} Throws an error if multiple providers are set or no provider is set
- */
-function getProvider(ttsSchema) {
- if (!ttsSchema) {
- throw new Error(`No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)?
-
- https://www.librechat.ai/docs/configuration/stt_tts#tts`);
- }
- const providers = Object.entries(ttsSchema).filter(([, value]) => Object.keys(value).length > 0);
-
- if (providers.length > 1) {
- throw new Error('Multiple providers are set. Please set only one provider.');
- } else if (providers.length === 0) {
- throw new Error('No provider is set. Please set a provider.');
- } else {
- return providers[0][0];
- }
-}
-
-/**
- * removeUndefined function
- * This function takes an object and removes all keys with undefined values
- * It also removes keys with empty objects as values
- *
- * @param {Object} obj - The object to be cleaned
- * @returns {void} This function does not return a value. It modifies the input object directly
- */
-function removeUndefined(obj) {
- Object.keys(obj).forEach((key) => {
- if (obj[key] && typeof obj[key] === 'object') {
- removeUndefined(obj[key]);
- if (Object.keys(obj[key]).length === 0) {
- delete obj[key];
- }
- } else if (obj[key] === undefined) {
- delete obj[key];
- }
- });
-}
-
-/**
- * This function prepares the necessary data and headers for making a request to the OpenAI TTS
- * It uses the provided TTS schema, input text, and voice to create the request
- *
- * @param {TCustomConfig['tts']['openai']} ttsSchema - The TTS schema containing the OpenAI configuration
- * @param {string} input - The text to be converted to speech
- * @param {string} voice - The voice to be used for the speech
- *
- * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
- * If an error occurs, it throws an error with a message indicating that the selected voice is not available
- */
-function openAIProvider(ttsSchema, input, voice) {
- const url = ttsSchema?.url || 'https://api.openai.com/v1/audio/speech';
-
- if (
- ttsSchema?.voices &&
- ttsSchema.voices.length > 0 &&
- !ttsSchema.voices.includes(voice) &&
- !ttsSchema.voices.includes('ALL')
- ) {
- throw new Error(`Voice ${voice} is not available.`);
- }
-
- let data = {
- input,
- model: ttsSchema?.model,
- voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,
- backend: ttsSchema?.backend,
- };
-
- let headers = {
- 'Content-Type': 'application/json',
- Authorization: 'Bearer ' + extractEnvVariable(ttsSchema?.apiKey),
- };
-
- [data, headers].forEach(removeUndefined);
-
- return [url, data, headers];
-}
-
-/**
- * Generates the necessary parameters for making a request to Azure's OpenAI Text-to-Speech API.
- *
- * @param {TCustomConfig['tts']['azureOpenAI']} ttsSchema - The TTS schema containing the AzureOpenAI configuration
- * @param {string} input - The text to be converted to speech
- * @param {string} voice - The voice to be used for the speech
- *
- * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
- * If an error occurs, it throws an error with a message indicating that the selected voice is not available
- */
-function azureOpenAIProvider(ttsSchema, input, voice) {
- const instanceName = ttsSchema?.instanceName;
- const deploymentName = ttsSchema?.deploymentName;
- const apiVersion = ttsSchema?.apiVersion;
-
- const url =
- genAzureEndpoint({
- azureOpenAIApiInstanceName: instanceName,
- azureOpenAIApiDeploymentName: deploymentName,
- }) +
- '/audio/speech?api-version=' +
- apiVersion;
-
- const apiKey = ttsSchema.apiKey ? extractEnvVariable(ttsSchema.apiKey) : '';
-
- if (
- ttsSchema?.voices &&
- ttsSchema.voices.length > 0 &&
- !ttsSchema.voices.includes(voice) &&
- !ttsSchema.voices.includes('ALL')
- ) {
- throw new Error(`Voice ${voice} is not available.`);
- }
-
- let data = {
- model: ttsSchema?.model,
- input,
- voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,
- };
-
- let headers = {
- 'Content-Type': 'application/json',
- };
-
- [data, headers].forEach(removeUndefined);
-
- if (apiKey) {
- headers['api-key'] = apiKey;
- }
-
- return [url, data, headers];
-}
-
-/**
- * elevenLabsProvider function
- * This function prepares the necessary data and headers for making a request to the Eleven Labs TTS
- * It uses the provided TTS schema, input text, and voice to create the request
- *
- * @param {TCustomConfig['tts']['elevenLabs']} ttsSchema - The TTS schema containing the Eleven Labs configuration
- * @param {string} input - The text to be converted to speech
- * @param {string} voice - The voice to be used for the speech
- * @param {boolean} stream - Whether to stream the audio or not
- *
- * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
- * @throws {Error} Throws an error if the selected voice is not available
- */
-function elevenLabsProvider(ttsSchema, input, voice, stream) {
- let url =
- ttsSchema?.url ||
- `https://api.elevenlabs.io/v1/text-to-speech/{voice_id}${stream ? '/stream' : ''}`;
-
- if (!ttsSchema?.voices.includes(voice) && !ttsSchema?.voices.includes('ALL')) {
- throw new Error(`Voice ${voice} is not available.`);
- }
-
- url = url.replace('{voice_id}', voice);
-
- let data = {
- model_id: ttsSchema?.model,
- text: input,
- // voice_id: voice,
- voice_settings: {
- similarity_boost: ttsSchema?.voice_settings?.similarity_boost,
- stability: ttsSchema?.voice_settings?.stability,
- style: ttsSchema?.voice_settings?.style,
- use_speaker_boost: ttsSchema?.voice_settings?.use_speaker_boost || undefined,
- },
- pronunciation_dictionary_locators: ttsSchema?.pronunciation_dictionary_locators,
- };
-
- let headers = {
- 'Content-Type': 'application/json',
- 'xi-api-key': extractEnvVariable(ttsSchema?.apiKey),
- Accept: 'audio/mpeg',
- };
-
- [data, headers].forEach(removeUndefined);
-
- return [url, data, headers];
-}
-
-/**
- * localAIProvider function
- * This function prepares the necessary data and headers for making a request to the LocalAI TTS
- * It uses the provided TTS schema, input text, and voice to create the request
- *
- * @param {TCustomConfig['tts']['localai']} ttsSchema - The TTS schema containing the LocalAI configuration
- * @param {string} input - The text to be converted to speech
- * @param {string} voice - The voice to be used for the speech
- *
- * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
- * @throws {Error} Throws an error if the selected voice is not available
- */
-function localAIProvider(ttsSchema, input, voice) {
- let url = ttsSchema?.url;
-
- if (
- ttsSchema?.voices &&
- ttsSchema.voices.length > 0 &&
- !ttsSchema.voices.includes(voice) &&
- !ttsSchema.voices.includes('ALL')
- ) {
- throw new Error(`Voice ${voice} is not available.`);
- }
-
- let data = {
- input,
- model: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined,
- backend: ttsSchema?.backend,
- };
-
- let headers = {
- 'Content-Type': 'application/json',
- Authorization: 'Bearer ' + extractEnvVariable(ttsSchema?.apiKey),
- };
-
- [data, headers].forEach(removeUndefined);
-
- if (extractEnvVariable(ttsSchema.apiKey) === '') {
- delete headers.Authorization;
- }
-
- return [url, data, headers];
-}
-
-/**
- *
- * Returns provider and its schema for use with TTS requests
- * @param {TCustomConfig} customConfig
- * @param {string} _voice
- * @returns {Promise<[string, TProviderSchema]>}
- */
-async function getProviderSchema(customConfig) {
- const provider = getProvider(customConfig.speech.tts);
- return [provider, customConfig.speech.tts[provider]];
-}
-
-/**
- *
- * Returns a tuple of the TTS schema as well as the voice for the TTS request
- * @param {TProviderSchema} providerSchema
- * @param {string} requestVoice
- * @returns {Promise<string>}
- */
-async function getVoice(providerSchema, requestVoice) {
- const voices = providerSchema.voices.filter((voice) => voice && voice.toUpperCase() !== 'ALL');
- let voice = requestVoice;
- if (!voice || !voices.includes(voice) || (voice.toUpperCase() === 'ALL' && voices.length > 1)) {
- voice = getRandomVoiceId(voices);
- }
-
- return voice;
-}
-
-/**
- *
- * @param {string} provider
- * @param {TProviderSchema} ttsSchema
- * @param {object} params
- * @param {string} params.voice
- * @param {string} params.input
- * @param {boolean} [params.stream]
- * @returns {Promise<ArrayBuffer>}
- */
-async function ttsRequest(provider, ttsSchema, { input, voice, stream = true } = { stream: true }) {
- let [url, data, headers] = [];
- switch (provider) {
- case TTSProviders.OPENAI:
- [url, data, headers] = openAIProvider(ttsSchema, input, voice);
- break;
- case TTSProviders.AZURE_OPENAI:
- [url, data, headers] = azureOpenAIProvider(ttsSchema, input, voice);
- break;
- case TTSProviders.ELEVENLABS:
- [url, data, headers] = elevenLabsProvider(ttsSchema, input, voice, stream);
- break;
- case TTSProviders.LOCALAI:
- [url, data, headers] = localAIProvider(ttsSchema, input, voice);
- break;
- default:
- throw new Error('Invalid provider');
- }
-
- if (stream) {
- return await axios.post(url, data, { headers, responseType: 'stream' });
- }
-
- return await axios.post(url, data, { headers, responseType: 'arraybuffer' });
-}
-
-/**
- * Handles a text-to-speech request. Extracts input and voice from the request, retrieves the TTS configuration,
- * and sends a request to the appropriate provider. The resulting audio data is sent in the response
- *
- * @param {Object} req - The request object, which should contain the input text and voice in its body
- * @param {Object} res - The response object, used to send the audio data or an error message
- *
- * @returns {Promise<void>} This function does not return a value. It sends the audio data or an error message in the response
- *
- * @throws {Error} Throws an error if the provider is invalid
- */
-async function textToSpeech(req, res) {
- const { input } = req.body;
-
- if (!input) {
- return res.status(400).send('Missing text in request body');
- }
-
- const customConfig = await getCustomConfig();
- if (!customConfig) {
- res.status(500).send('Custom config not found');
- }
-
- try {
- res.setHeader('Content-Type', 'audio/mpeg');
- const [provider, ttsSchema] = await getProviderSchema(customConfig);
- const voice = await getVoice(ttsSchema, req.body.voice);
- if (input.length < 4096) {
- const response = await ttsRequest(provider, ttsSchema, { input, voice });
- response.data.pipe(res);
- return;
- }
-
- const textChunks = splitTextIntoChunks(input, 1000);
-
- for (const chunk of textChunks) {
- try {
- const response = await ttsRequest(provider, ttsSchema, {
- voice,
- input: chunk.text,
- stream: true,
- });
-
- logger.debug(`[textToSpeech] user: ${req?.user?.id} | writing audio stream`);
- await new Promise((resolve) => {
- response.data.pipe(res, { end: chunk.isFinished });
- response.data.on('end', () => {
- resolve();
- });
- });
-
- if (chunk.isFinished) {
- break;
- }
- } catch (innerError) {
- logger.error('Error processing manual update:', chunk, innerError);
- if (!res.headersSent) {
- res.status(500).end();
- }
- return;
- }
- }
-
- if (!res.headersSent) {
- res.end();
- }
- } catch (error) {
- logger.error(
- 'Error creating the audio stream. Suggestion: check your provider quota. Error:',
- error,
- );
- res.status(500).send('An error occurred');
- }
-}
-
-async function streamAudio(req, res) {
- res.setHeader('Content-Type', 'audio/mpeg');
- const customConfig = await getCustomConfig();
- if (!customConfig) {
- return res.status(500).send('Custom config not found');
- }
-
- const [provider, ttsSchema] = await getProviderSchema(customConfig);
- const voice = await getVoice(ttsSchema, req.body.voice);
-
- try {
- let shouldContinue = true;
-
- req.on('close', () => {
- logger.warn('[streamAudio] Audio Stream Request closed by client');
- shouldContinue = false;
- });
-
- const processChunks = createChunkProcessor(req.body.messageId);
-
- while (shouldContinue) {
- // example updates
- // const updates = [
- // { text: 'This is a test.', isFinished: false },
- // { text: 'This is only a test.', isFinished: false },
- // { text: 'Your voice is like a combination of Fergie and Jesus!', isFinished: true },
- // ];
-
- const updates = await processChunks();
- if (typeof updates === 'string') {
- logger.error(`Error processing audio stream updates: ${JSON.stringify(updates)}`);
- res.status(500).end();
- return;
- }
-
- if (updates.length === 0) {
- await new Promise((resolve) => setTimeout(resolve, 1250));
- continue;
- }
-
- for (const update of updates) {
- try {
- const response = await ttsRequest(provider, ttsSchema, {
- voice,
- input: update.text,
- stream: true,
- });
-
- if (!shouldContinue) {
- break;
- }
-
- logger.debug(`[streamAudio] user: ${req?.user?.id} | writing audio stream`);
- await new Promise((resolve) => {
- response.data.pipe(res, { end: update.isFinished });
- response.data.on('end', () => {
- resolve();
- });
- });
-
- if (update.isFinished) {
- shouldContinue = false;
- break;
- }
- } catch (innerError) {
- logger.error('Error processing audio stream update:', update, innerError);
- if (!res.headersSent) {
- res.status(500).end();
- }
- return;
- }
- }
-
- if (!shouldContinue) {
- break;
- }
- }
-
- if (!res.headersSent) {
- res.end();
- }
- } catch (error) {
- logger.error('Failed to fetch audio:', error);
- if (!res.headersSent) {
- res.status(500).end();
- }
- }
-}
-
-module.exports = {
- textToSpeech,
- getProvider,
- streamAudio,
-};
diff --git a/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx b/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx
index 30ff66dc0d..12836775e5 100755
--- a/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx
+++ b/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx
@@ -15,7 +15,7 @@ export default function DecibelSelector() {
<div className="flex items-center justify-between">
<div>{localize('com_nav_db_sensitivity')}</div>
<div className="w-2" />
- <small className="opacity-40">({localize('com_endpoint_default_with_num', '0.45')})</small>
+ <small className="opacity-40">({localize('com_endpoint_default_with_num', '-45')})</small>
</div>
<div className="flex items-center justify-between">
<Slider
diff --git a/client/src/hooks/Input/useSpeechToTextExternal.ts b/client/src/hooks/Input/useSpeechToTextExternal.ts
index 371d5d22a2..01ab107817 100644
--- a/client/src/hooks/Input/useSpeechToTextExternal.ts
+++ b/client/src/hooks/Input/useSpeechToTextExternal.ts
@@ -45,7 +45,9 @@ const useSpeechToTextExternal = (onTranscriptionComplete: (text: string) => void
const cleanup = () => {
if (mediaRecorderRef.current) {
- mediaRecorderRef.current.removeEventListener('dataavailable', handleDataAvailable);
+ mediaRecorderRef.current.removeEventListener('dataavailable', (event: BlobEvent) => {
+ audioChunks.push(event.data);
+ });
mediaRecorderRef.current.removeEventListener('stop', handleStop);
mediaRecorderRef.current = null;
}
@@ -68,14 +70,6 @@ const useSpeechToTextExternal = (onTranscriptionComplete: (text: string) => void
}
};
- const handleDataAvailable = (event: BlobEvent) => {
- if (event.data.size > 0) {
- audioChunks.push(event.data);
- } else {
- showToast({ message: 'No audio data available', status: 'warning' });
- }
- };
-
const handleStop = () => {
if (audioChunks.length > 0) {
const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
@@ -139,7 +133,9 @@ const useSpeechToTextExternal = (onTranscriptionComplete: (text: string) => void
try {
setAudioChunks([]);
mediaRecorderRef.current = new MediaRecorder(audioStream.current);
- mediaRecorderRef.current.addEventListener('dataavailable', handleDataAvailable);
+ mediaRecorderRef.current.addEventListener('dataavailable', (event: BlobEvent) => {
+ audioChunks.push(event.data);
+ });
mediaRecorderRef.current.addEventListener('stop', handleStop);
mediaRecorderRef.current.start(100);
if (!audioContextRef.current && autoTranscribeAudio && speechToText) {