Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

4.0 KB
Raw Blame
import { Providers } from '@librechat/agents';
import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider';
import type { IMongoFile } from '@librechat/data-schemas';
import type { ServerRequest, StrategyFunctions, AudioResult } from '~/types';
import {
  getFileStream,
  getAudioFormat,
  getConfiguredFileSizeLimit,
  isAttachmentObjectNotFoundError,
  isConfiguredProviderMediaType,
} from './utils';
import { validateAudio } from '~/files/validation';
import { runGuardedEncode } from './memoryGuard';

/**
 * Encodes and formats audio files for different providers
 * @param req - The request object
 * @param files - Array of audio files
 * @param params - Object containing provider and optional endpoint
 * @param params.provider - The provider to format for
 * @param params.endpoint - Optional endpoint name for file config lookup
 * @param getStrategyFunctions - Function to get strategy functions
 * @returns Promise that resolves to audio and file metadata
 */
export async function encodeAndFormatAudios(
  req: ServerRequest,
  files: IMongoFile[],
  params: { provider: Providers; endpoint?: string },
  getStrategyFunctions: (source: string) => StrategyFunctions,
): Promise<AudioResult> {
  const { provider, endpoint } = params;
  if (!files?.length) {
    return { audios: [], files: [] };
  }

  const encodingMethods: Record<string, StrategyFunctions> = {};
  const result: AudioResult = { audios: [], files: [] };

  const results = await Promise.allSettled(
    files.map((file) =>
      runGuardedEncode(file.bytes ?? 0, () =>
        getFileStream(req, file, encodingMethods, getStrategyFunctions),
      ),
    ),
  );

  for (const settledResult of results) {
    if (settledResult.status === 'rejected') {
      if (isAttachmentObjectNotFoundError(settledResult.reason)) {
        throw settledResult.reason;
      }
      console.error('Audio processing failed:', settledResult.reason);
      continue;
    }

    const processed = settledResult.value;
    if (!processed) continue;

    const { file, content, metadata } = processed;

    if (!content || !file) {
      if (metadata) result.files.push(metadata);
      continue;
    }

    if (!file.type.startsWith('audio/') || !isDocumentSupportedProvider(provider)) {
      result.files.push(metadata);
      continue;
    }

    const audioBuffer = Buffer.from(content, 'base64');

    /** Extract configured file size limit from fileConfig for this endpoint */
    const configuredFileSizeLimit = getConfiguredFileSizeLimit(req, {
      provider,
      endpoint,
    });

    const validation = await validateAudio(
      audioBuffer,
      audioBuffer.length,
      provider,
      configuredFileSizeLimit,
    );

    if (!validation.isValid) {
      throw new Error(`Audio validation failed: ${validation.error}`);
    }

    if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {
      result.audios.push({
        type: 'media',
        mimeType: file.type,
        data: content,
      });
    } else if (
      provider === Providers.OPENROUTER ||
      (isOpenAILikeProvider(provider) &&
        isConfiguredProviderMediaType(req, { provider, endpoint }, file.type))
    ) {
      /** OpenAI-compatible `input_audio` part (OpenRouter, vLLM, and other gateways).
       * Custom endpoints only get here when the admin listed audio types in the
       * endpoint's `supportedMimeTypes`, since not every gateway accepts audio. */
      /** Canonicalized from the MIME type: the accepted MIME list contains aliases
       * (`audio/wave`, `audio/mpeg`) that are not valid format values, and the filename
       * may carry no extension at all. */
      const format = getAudioFormat(file.type, file.filename);
      if (!format) {
        throw new Error(
          `Could not determine a supported audio format for "${file.filename}" (${file.type})`,
        );
      }
      result.audios.push({
        type: 'input_audio',
        input_audio: {
          data: content,
          format,
        },
      });
    }

    result.files.push(metadata);
  }

  return result;
}