Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

7.8 KB
Raw Blame
import {
  FileSources,
  VisionModes,
  ImageDetail,
  ContentTypes,
  EModelEndpoint,
  mergeFileConfig,
  getEndpointFileConfig,
} from 'librechat-data-provider';
import type { IMongoFile } from '@librechat/data-schemas';
import type { Agents } from 'librechat-data-provider';
import type { AxiosInstance } from 'axios';
import type { ServerRequest, StrategyFunctions } from '~/types';
import { getOutboundTimeoutMs, outboundRequestBudget } from '~/utils/timeouts';
import { getFileStream, isAttachmentObjectNotFoundError } from './utils';
import { validateImage } from '~/files/validation';
import { sniffImageMimeType } from '~/files/mime';
import { runGuardedEncode } from './memoryGuard';
import { logAxiosError } from '~/utils/axios';

type ImageEncodingFile = Pick<
  IMongoFile,
  | 'file_id'
  | 'temp_file_id'
  | 'filepath'
  | 'storageKey'
  | 'filename'
  | 'type'
  | 'height'
  | 'width'
  | 'embedded'
  | 'metadata'
> &
  Partial<Pick<IMongoFile, 'source' | 'bytes'>>;

type ImageEntry = [ImageEncodingFile, string | null];

interface ImageStrategy extends StrategyFunctions {
  prepareImagePayload?: (
    req: ServerRequest,
    file: ImageEncodingFile,
  ) => Promise<[ImageEncodingFile, string]>;
}

interface ImageEncodingDependencies {
  getStrategyFunctions: (source: string) => ImageStrategy;
  httpClient: Pick<AxiosInstance, 'get'>;
}

interface ImageResult {
  files: Array<
    Pick<
      ImageEncodingFile,
      'file_id' | 'filepath' | 'filename' | 'type' | 'height' | 'width' | 'embedded' | 'metadata'
    >
  >;
  image_urls: Array<
    | Agents.MessageContentImageUrl
    | { type: ContentTypes.IMAGE_URL; inlineData: { mimeType: string; data: string } }
    | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
  >;
}

const base64Only = new Set<string>([
  EModelEndpoint.google,
  EModelEndpoint.anthropic,
  'Ollama',
  'ollama',
  EModelEndpoint.bedrock,
]);

const blobStorageSources = new Set<string>([
  FileSources.azure_blob,
  FileSources.s3,
  FileSources.firebase,
  FileSources.cloudfront,
]);

export class AttachmentStorageError extends Error {
  readonly code = 'ATTACHMENT_STORAGE_ERROR';

  constructor() {
    super('An attached file could not be read from storage. Try again or upload it again.');
    this.name = 'AttachmentStorageError';
  }
}

/** Owns source selection, guarded reads, validation and provider-specific image formatting. */
export async function encodeAndFormatImages(
  req: ServerRequest,
  files: ImageEncodingFile[] | null | undefined,
  params: { provider?: string; endpoint?: string; imageDetail?: Agents.ImageDetail },
  { getStrategyFunctions, httpClient }: ImageEncodingDependencies,
  mode?: string,
): Promise<ImageResult> {
  const { provider, endpoint } = params;
  const effectiveEndpoint = endpoint ?? provider;
  const promises: Array<ImageEntry | Promise<ImageEntry>> = [];
  const encodingMethods: Record<string, ImageStrategy> = {};
  const result: ImageResult = { files: [], image_urls: [] };

  if (!files?.length) {
    return result;
  }

  /** A later synchronous throw abandons earlier in-flight reads; keep their rejections handled. */
  const track = (entry: Promise<ImageEntry>): Promise<ImageEntry> => {
    entry.catch(() => undefined);
    return entry;
  };

  for (const file of files) {
    const source = file.source ?? FileSources.local;
    if (!file.height) {
      promises.push([file, null]);
      continue;
    }

    if (blobStorageSources.has(source)) {
      promises.push(
        track(
          runGuardedEncode(file.bytes ?? 0, () =>
            getFileStream(req, file, encodingMethods, getStrategyFunctions),
          ).then(
            (processedFile): ImageEntry => [file, processedFile?.content ?? null],
            (error: unknown): never => {
              if (isAttachmentObjectNotFoundError(error)) {
                throw error;
              }
              if (typeof error === 'object' && error != null && 'bufferedData' in error) {
                delete (error as { bufferedData?: unknown }).bufferedData;
              }
              throw new AttachmentStorageError();
            },
          ),
        ),
      );
      continue;
    }

    if (!encodingMethods[source]) {
      encodingMethods[source] = getStrategyFunctions(source);
    }
    const preparePayload = encodingMethods[source].prepareImagePayload;
    if (!preparePayload) {
      throw new Error(`Encoding function not implemented for ${source}`);
    }

    if (source !== FileSources.local && effectiveEndpoint && base64Only.has(effectiveEndpoint)) {
      const entry = runGuardedEncode<ImageEntry>(file.bytes ?? 0, async () => {
        const [preparedFile, imageURL] = await preparePayload(req, file);
        const fetchTimeoutMs = getOutboundTimeoutMs(req.config, 'imageFetchMs');
        try {
          const response = await httpClient.get<ArrayBuffer>(imageURL, {
            responseType: 'arraybuffer',
            ...outboundRequestBudget(fetchTimeoutMs, 'imageFetchMs'),
          });
          return [preparedFile, Buffer.from(response.data).toString('base64')];
        } catch (error) {
          const message = 'Error fetching image to convert to base64';
          throw new Error(logAxiosError({ message, error }));
        }
      });
      promises.push(track(entry));
      continue;
    }
    promises.push(track(preparePayload(req, file)));
  }

  const detail = params.imageDetail ?? req.body.imageDetail ?? ImageDetail.auto;
  const formattedImages = await Promise.all(promises);
  let configuredFileSizeLimit: number | undefined;
  if (req.config?.fileConfig) {
    const fileConfig = mergeFileConfig(req.config.fileConfig);
    const endpointConfig = getEndpointFileConfig({
      fileConfig,
      endpoint: effectiveEndpoint,
    });
    configuredFileSizeLimit = endpointConfig?.fileSizeLimit;
  }

  for (const [file, imageContent] of formattedImages) {
    const fileMetadata: ImageResult['files'][number] = {
      type: file.type,
      file_id: file.file_id,
      filepath: file.filepath,
      filename: file.filename,
      embedded: !!file.embedded,
      metadata: file.metadata,
    };
    if (file.height && file.width) {
      fileMetadata.height = file.height;
      fileMetadata.width = file.width;
    }
    result.files.push(fileMetadata);

    if (!imageContent) {
      continue;
    }

    const isURL = imageContent.startsWith('http');
    // Historical uploads may have a stored type that disagrees with their bytes.
    let mediaType = file.type;
    if (!isURL) {
      const imageBuffer = Buffer.from(imageContent, 'base64');
      mediaType = sniffImageMimeType(imageBuffer) ?? file.type;
      if (file.height && file.width) {
        const validation = await validateImage(
          imageBuffer,
          imageBuffer.length,
          effectiveEndpoint ?? '',
          configuredFileSizeLimit,
        );
        if (!validation.isValid) {
          throw new Error(`Image validation failed for ${file.filename}: ${validation.error}`);
        }
      }
    }

    const url = isURL ? imageContent : `data:${mediaType};base64,${imageContent}`;
    if (mode === VisionModes.agents) {
      result.image_urls.push({ type: ContentTypes.IMAGE_URL, image_url: { url, detail } });
    } else if (effectiveEndpoint === EModelEndpoint.google && mode === VisionModes.generative) {
      result.image_urls.push({
        type: ContentTypes.IMAGE_URL,
        inlineData: { mimeType: mediaType, data: imageContent },
      });
    } else if (effectiveEndpoint === EModelEndpoint.google) {
      result.image_urls.push({ type: ContentTypes.IMAGE_URL, image_url: url });
    } else if (effectiveEndpoint === EModelEndpoint.anthropic) {
      result.image_urls.push({
        type: 'image',
        source: { type: 'base64', media_type: mediaType, data: imageContent },
      });
    } else {
      result.image_urls.push({ type: ContentTypes.IMAGE_URL, image_url: { url, detail } });
    }
  }
  return result;
}