Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

14.7 KB
Raw Blame
import sharp from 'sharp';
import { Readable } from 'node:stream';
import { FileSources, VisionModes } from 'librechat-data-provider';
import type { ServerRequest } from '~/types';
import { AttachmentStorageError, encodeAndFormatImages } from './image';
import * as memoryGuard from './memoryGuard';

const imageBytes = Buffer.from('image-content-bytes');
const content = imageBytes.toString('base64');
const dataUrl = `data:image/png;base64,${content}`;
const file = {
  file_id: 'image-1',
  filepath: 'https://storage.example/image.png?signature=secret',
  storageKey: 'images/user/image.png',
  filename: 'image.png',
  type: 'image/png',
  bytes: imageBytes.length,
  height: 10,
  width: 10,
  source: FileSources.s3,
};
const makeReq = (config: Partial<NonNullable<ServerRequest['config']>> = {}) =>
  ({ body: {}, config }) as ServerRequest;

describe('encodeAndFormatImages', () => {
  const getDownloadStream = jest.fn();
  const prepareImagePayload = jest.fn();
  const getStrategyFunctions: jest.MockedFunction<
    Parameters<typeof encodeAndFormatImages>[3]['getStrategyFunctions']
  > = jest.fn();
  const httpClient = { get: jest.fn() };
  const deps = { getStrategyFunctions, httpClient };

  beforeEach(() => {
    jest.resetAllMocks();
    getDownloadStream.mockImplementation(async () => Readable.from(imageBytes));
    prepareImagePayload.mockImplementation(async (_req, image) => [image, content]);
    getStrategyFunctions.mockImplementation(() => ({ getDownloadStream, prepareImagePayload }));
    httpClient.get.mockResolvedValue({ data: imageBytes });
    jest.spyOn(memoryGuard, 'runGuardedEncode');
  });

  afterEach(() => jest.restoreAllMocks());

  it.each([FileSources.s3, FileSources.cloudfront, FileSources.azure_blob, FileSources.firebase])(
    'reads %s by canonical key without entering the legacy payload path',
    async (source) => {
      const result = await encodeAndFormatImages(makeReq(), [{ ...file, source }], {}, deps);

      expect(result.image_urls).toEqual([
        { type: 'image_url', image_url: { url: dataUrl, detail: 'auto' } },
      ]);
      expect(result.files).toEqual([
        expect.objectContaining({ file_id: file.file_id, height: 10, width: 10, embedded: false }),
      ]);
      expect(result.files[0]).not.toHaveProperty('storageKey');
      expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), file.storageKey);
      expect(memoryGuard.runGuardedEncode).toHaveBeenCalledWith(file.bytes, expect.any(Function));
      expect(prepareImagePayload).not.toHaveBeenCalled();
      expect(httpClient.get).not.toHaveBeenCalled();
    },
  );

  it('uses the legacy filepath when no canonical key was stored', async () => {
    await encodeAndFormatImages(makeReq(), [{ ...file, storageKey: undefined }], {}, deps);
    expect(getDownloadStream).toHaveBeenCalledWith(expect.anything(), file.filepath);
  });

  it('overlaps storage reads while keeping attachment order and the first error in order', async () => {
    let active = 0;
    let peak = 0;
    getDownloadStream.mockImplementation(async (_req, key: string) => {
      active++;
      peak = Math.max(peak, active);
      await new Promise((resolve) => setTimeout(resolve, 5));
      active--;
      if (key === 'b') {
        throw new Error('second failed');
      }
      return Readable.from(imageBytes);
    });
    const files = ['a', 'c', 'd'].map((key) => ({ ...file, file_id: key, storageKey: key }));

    const result = await encodeAndFormatImages(makeReq(), files, {}, deps);
    expect(result.files.map((f) => f.file_id)).toEqual(['a', 'c', 'd']);
    expect(peak).toBeGreaterThan(1);

    await expect(
      encodeAndFormatImages(
        makeReq(),
        [files[0], { ...file, file_id: 'b', storageKey: 'b' }, files[1]],
        {},
        deps,
      ),
    ).rejects.toBeInstanceOf(AttachmentStorageError);
  });

  it('surfaces an early storage failure without waiting for a stalled later read', async () => {
    getDownloadStream.mockImplementation(async (_req, key: string) => {
      if (key === 'stalled') {
        return new Promise(() => undefined);
      }
      throw new Error('first failed');
    });
    const files = ['early', 'stalled'].map((key) => ({ ...file, file_id: key, storageKey: key }));

    await expect(encodeAndFormatImages(makeReq(), files, {}, deps)).rejects.toBeInstanceOf(
      AttachmentStorageError,
    );
  });

  it('surfaces a later failure without waiting for an earlier pending read', async () => {
    getDownloadStream.mockImplementation(async (_req, key: string) => {
      if (key === 'slow') {
        return new Promise(() => undefined);
      }
      throw new Error('later failed');
    });
    const files = ['slow', 'fails'].map((key) => ({ ...file, file_id: key, storageKey: key }));

    await expect(encodeAndFormatImages(makeReq(), files, {}, deps)).rejects.toBeInstanceOf(
      AttachmentStorageError,
    );
  });

  it('returns an empty result without acquiring a strategy for absent files', async () => {
    await expect(encodeAndFormatImages(makeReq(), undefined, {}, deps)).resolves.toEqual({
      files: [],
      image_urls: [],
    });
    await expect(encodeAndFormatImages(makeReq(), [], {}, deps)).resolves.toEqual({
      files: [],
      image_urls: [],
    });
    expect(getStrategyFunctions).not.toHaveBeenCalled();
  });

  it('keeps metadata-only attachments without reading storage', async () => {
    const result = await encodeAndFormatImages(
      makeReq(),
      [{ ...file, height: undefined }],
      {},
      deps,
    );
    expect(result.files).toHaveLength(1);
    expect(result.image_urls).toEqual([]);
    expect(getStrategyFunctions).not.toHaveBeenCalled();
  });

  it('keeps metadata when a stored image has no download reference', async () => {
    const result = await encodeAndFormatImages(
      makeReq(),
      [{ ...file, filepath: '', storageKey: undefined }],
      {},
      deps,
    );
    expect(result.files).toHaveLength(1);
    expect(result.image_urls).toEqual([]);
    expect(getDownloadStream).not.toHaveBeenCalled();
    expect(prepareImagePayload).not.toHaveBeenCalled();
  });

  it('preserves the missing-object error without falling back to an external URL', async () => {
    getDownloadStream.mockRejectedValue({ name: 'NoSuchKey' });
    await expect(encodeAndFormatImages(makeReq(), [file], {}, deps)).rejects.toMatchObject({
      code: 'ATTACHMENT_OBJECT_NOT_FOUND',
      fileId: file.file_id,
    });
    expect(prepareImagePayload).not.toHaveBeenCalled();
    expect(httpClient.get).not.toHaveBeenCalled();
  });

  it('discards buffered image bytes and hides storage error details', async () => {
    const failure = new Error('private storage path');
    getDownloadStream.mockResolvedValue(
      new Readable({
        read() {
          this.push(imageBytes);
          this.destroy(failure);
        },
      }),
    );
    await expect(encodeAndFormatImages(makeReq(), [file], {}, deps)).rejects.toEqual(
      new AttachmentStorageError(),
    );
    expect(failure).not.toHaveProperty('bufferedData');
    expect(prepareImagePayload).not.toHaveBeenCalled();
    expect(httpClient.get).not.toHaveBeenCalled();
  });

  it.each([FileSources.local, undefined])(
    'keeps the unguarded local path for %s',
    async (source) => {
      const result = await encodeAndFormatImages(makeReq(), [{ ...file, source }], {}, deps);
      expect(result.image_urls).toEqual([
        { type: 'image_url', image_url: { url: dataUrl, detail: 'auto' } },
      ]);
      expect(prepareImagePayload).toHaveBeenCalled();
      expect(getDownloadStream).not.toHaveBeenCalled();
      expect(memoryGuard.runGuardedEncode).not.toHaveBeenCalled();
    },
  );

  it('keeps legacy external URLs for endpoints that accept them', async () => {
    prepareImagePayload.mockImplementation(async (_req, image) => [image, file.filepath]);
    const result = await encodeAndFormatImages(
      makeReq(),
      [{ ...file, source: FileSources.openai }],
      { endpoint: 'openai' },
      deps,
    );
    expect(result.image_urls).toEqual([
      { type: 'image_url', image_url: { url: file.filepath, detail: 'auto' } },
    ]);
    expect(httpClient.get).not.toHaveBeenCalled();
    expect(memoryGuard.runGuardedEncode).not.toHaveBeenCalled();
  });

  it('guards legacy URL downloads for base64-only endpoints', async () => {
    prepareImagePayload.mockImplementation(async (_req, image) => [image, file.filepath]);
    const result = await encodeAndFormatImages(
      makeReq(),
      [{ ...file, source: FileSources.openai }],
      { endpoint: 'anthropic' },
      deps,
    );
    expect(result.image_urls).toEqual([
      { type: 'image', source: { type: 'base64', media_type: file.type, data: content } },
    ]);
    expect(httpClient.get).toHaveBeenCalledWith(file.filepath, {
      responseType: 'arraybuffer',
      timeout: 30_000,
      signal: expect.any(AbortSignal),
      timeoutErrorMessage: expect.any(String),
    });
    expect(memoryGuard.runGuardedEncode).toHaveBeenCalledWith(file.bytes, expect.any(Function));
  });

  it('rejects unsupported legacy strategies instead of silently dropping images', async () => {
    getStrategyFunctions.mockReturnValue({ getDownloadStream });
    await expect(
      encodeAndFormatImages(makeReq(), [{ ...file, source: FileSources.openai }], {}, deps),
    ).rejects.toThrow('Encoding function not implemented for openai');
  });

  it.each([
    [
      'google',
      VisionModes.generative,
      { type: 'image_url', inlineData: { mimeType: file.type, data: content } },
    ],
    ['google', undefined, { type: 'image_url', image_url: dataUrl }],
    [
      'anthropic',
      undefined,
      { type: 'image', source: { type: 'base64', media_type: file.type, data: content } },
    ],
    [
      'anthropic',
      VisionModes.agents,
      { type: 'image_url', image_url: { url: dataUrl, detail: 'auto' } },
    ],
  ])('preserves the %s payload in mode %s', async (provider, mode, expected) => {
    const result = await encodeAndFormatImages(makeReq(), [file], { provider }, deps, mode);
    expect(result.image_urls).toEqual([expected]);
  });

  describe('a record whose type does not describe its bytes', () => {
    let jpegBytes: Buffer;
    let jpegContent: string;
    const mislabeled = () => ({ ...file, bytes: jpegBytes.length, type: 'image/png' });

    beforeAll(async () => {
      jpegBytes = await sharp({
        create: { width: 8, height: 8, channels: 3, background: { r: 1, g: 2, b: 3 } },
      })
        .jpeg()
        .toBuffer();
      jpegContent = jpegBytes.toString('base64');
    });

    beforeEach(() => {
      getDownloadStream.mockImplementation(async () => Readable.from(jpegBytes));
    });

    it('declares the bytes to Anthropic, not the stored type', async () => {
      const result = await encodeAndFormatImages(
        makeReq(),
        [mislabeled()],
        { endpoint: 'anthropic' },
        deps,
      );

      expect(result.image_urls).toEqual([
        { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: jpegContent } },
      ]);
    });

    it('uses the real type for Claude conversations in agents mode', async () => {
      const result = await encodeAndFormatImages(
        makeReq(),
        [mislabeled()],
        { provider: 'anthropic' },
        deps,
        VisionModes.agents,
      );

      expect(result.image_urls).toEqual([
        {
          type: 'image_url',
          image_url: { url: `data:image/jpeg;base64,${jpegContent}`, detail: 'auto' },
        },
      ]);
    });

    it('declares the bytes to Google', async () => {
      const result = await encodeAndFormatImages(
        makeReq(),
        [mislabeled()],
        { provider: 'google' },
        deps,
        VisionModes.generative,
      );

      expect(result.image_urls).toEqual([
        { type: 'image_url', inlineData: { mimeType: 'image/jpeg', data: jpegContent } },
      ]);
    });

    it('declares the bytes in the data URL', async () => {
      const result = await encodeAndFormatImages(makeReq(), [mislabeled()], {}, deps);

      expect(result.image_urls).toEqual([
        {
          type: 'image_url',
          image_url: { url: `data:image/jpeg;base64,${jpegContent}`, detail: 'auto' },
        },
      ]);
    });

    it('keeps the stored type when the bytes name no format of their own', async () => {
      getDownloadStream.mockImplementation(async () => Readable.from(imageBytes));

      const result = await encodeAndFormatImages(
        makeReq(),
        [{ ...file, type: 'image/png' }],
        { endpoint: 'anthropic' },
        deps,
      );

      expect(result.image_urls).toEqual([
        { type: 'image', source: { type: 'base64', media_type: 'image/png', data: content } },
      ]);
    });

    it('leaves an external URL alone, having no bytes to read', async () => {
      prepareImagePayload.mockImplementation(async (_req, image) => [image, file.filepath]);

      const result = await encodeAndFormatImages(
        makeReq(),
        [{ ...mislabeled(), source: FileSources.openai }],
        { endpoint: 'openai' },
        deps,
      );

      expect(result.image_urls).toEqual([
        { type: 'image_url', image_url: { url: file.filepath, detail: 'auto' } },
      ]);
    });
  });

  it.each([
    [undefined, undefined, 'auto'],
    ['low', undefined, 'low'],
    [undefined, 'high', 'high'],
    ['low', 'high', 'high'],
  ] as const)(
    'resolves body detail %s and caller detail %s',
    async (bodyDetail, imageDetail, expected) => {
      const req = makeReq();
      req.body.imageDetail = bodyDetail;
      const result = await encodeAndFormatImages(req, [file], { imageDetail }, deps);
      expect(result.image_urls).toEqual([
        { type: 'image_url', image_url: { url: dataUrl, detail: expected } },
      ]);
    },
  );

  it('reuses each strategy and preserves order across storage and local images', async () => {
    const files = [
      file,
      { ...file, file_id: 'local', source: FileSources.local },
      { ...file, file_id: 'stored-2' },
    ];
    const result = await encodeAndFormatImages(makeReq(), files, {}, deps);
    expect(result.files.map((entry) => entry.file_id)).toEqual(files.map((entry) => entry.file_id));
    expect(result.image_urls).toHaveLength(3);
    expect(getStrategyFunctions).toHaveBeenCalledTimes(2);
    expect(getDownloadStream).toHaveBeenCalledTimes(2);
    expect(prepareImagePayload).toHaveBeenCalledTimes(1);
  });

  it.each([
    { endpoint: 'anthropic' },
    { provider: 'anthropic' },
    { provider: 'openai', endpoint: 'anthropic' },
  ])('resolves the configured size limit for %j', async (params) => {
    const req = makeReq({ fileConfig: { endpoints: { anthropic: { fileSizeLimit: 0.000001 } } } });
    await expect(encodeAndFormatImages(req, [file], params, deps)).rejects.toThrow(
      'Image validation failed for image.png: Image file size',
    );
  });
});