Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

fix: Match Image Media Types to Their Bytes (#17230)

lia-by-librechat[bot]520e5c05977d
diff --git a/api/server/services/Files/Azure/images.js b/api/server/services/Files/Azure/images.js
index 4d857d0d1d..1dc1877217 100644
--- a/api/server/services/Files/Azure/images.js
+++ b/api/server/services/Files/Azure/images.js
@@ -1,6 +1,7 @@
const fs = require('fs');
const path = require('path');
const sharp = require('sharp');
+const { convertImageBuffer } = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
const { resizeImageBuffer } = require('../images/resize');
const { updateUser, updateFile } = require('~/models');
@@ -37,23 +38,16 @@ async function uploadImageToAzure({
buffer: resizedBuffer,
width,
height,
+ type,
} = await resizeImageBuffer(inputBuffer, resolution, endpoint);
- const extension = path.extname(inputFilePath);
const userId = req.user.id;
- let webPBuffer;
- let fileName = `${file_id}__${path.basename(inputFilePath)}`;
- const targetExtension = `.${appConfig.imageOutputType}`;
+ const { buffer: webPBuffer, fileName } = await convertImageBuffer({
+ buffer: resizedBuffer,
+ type,
+ fileName: `${file_id}__${path.basename(inputFilePath)}`,
+ outputType: appConfig.imageOutputType,
+ });
- if (extension.toLowerCase() === targetExtension) {
- webPBuffer = resizedBuffer;
- } else {
- webPBuffer = await sharp(resizedBuffer).toFormat(appConfig.imageOutputType).toBuffer();
- const extRegExp = new RegExp(path.extname(fileName) + '$');
- fileName = fileName.replace(extRegExp, targetExtension);
- if (!path.extname(fileName)) {
- fileName += targetExtension;
- }
- }
const downloadURL = await saveBufferToAzure({
userId,
buffer: webPBuffer,
diff --git a/api/server/services/Files/Firebase/images.js b/api/server/services/Files/Firebase/images.js
index cffafa890c..7d04ea0047 100644
--- a/api/server/services/Files/Firebase/images.js
+++ b/api/server/services/Files/Firebase/images.js
@@ -1,6 +1,7 @@
const fs = require('fs');
const path = require('path');
const sharp = require('sharp');
+const { convertImageBuffer } = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
const { resizeImageBuffer } = require('../images/resize');
const { updateUser, updateFile } = require('~/models');
@@ -32,24 +33,15 @@ async function uploadImageToFirebase({ req, file, file_id, endpoint, resolution
buffer: resizedBuffer,
width,
height,
+ type,
} = await resizeImageBuffer(inputBuffer, resolution, endpoint);
- const extension = path.extname(inputFilePath);
const userId = req.user.id;
-
- let webPBuffer;
- let fileName = `${file_id}__${path.basename(inputFilePath)}`;
- const targetExtension = `.${appConfig.imageOutputType}`;
- if (extension.toLowerCase() === targetExtension) {
- webPBuffer = resizedBuffer;
- } else {
- webPBuffer = await sharp(resizedBuffer).toFormat(appConfig.imageOutputType).toBuffer();
- // Replace or append the correct extension
- const extRegExp = new RegExp(path.extname(fileName) + '$');
- fileName = fileName.replace(extRegExp, targetExtension);
- if (!path.extname(fileName)) {
- fileName += targetExtension;
- }
- }
+ const { buffer: webPBuffer, fileName } = await convertImageBuffer({
+ buffer: resizedBuffer,
+ type,
+ fileName: `${file_id}__${path.basename(inputFilePath)}`,
+ outputType: appConfig.imageOutputType,
+ });
const downloadURL = await saveBufferToFirebase({ userId, buffer: webPBuffer, fileName });
diff --git a/api/server/services/Files/Local/__tests__/images.spec.js b/api/server/services/Files/Local/__tests__/images.spec.js
index f64d547765..454e37e5d1 100644
--- a/api/server/services/Files/Local/__tests__/images.spec.js
+++ b/api/server/services/Files/Local/__tests__/images.spec.js
@@ -1,8 +1,3 @@
-jest.mock('sharp', () => ({}));
-jest.mock('@librechat/api', () => ({
- stripCacheBust: jest.fn((filepath) => filepath.split('?')[0]),
-}));
-jest.mock('../../images/resize', () => ({ resizeImageBuffer: jest.fn() }));
jest.mock('~/models', () => ({
updateUser: jest.fn(),
updateFile: jest.fn(async (doc) => doc),
@@ -11,8 +6,9 @@ jest.mock('~/models', () => ({
const fs = require('fs');
const os = require('os');
const path = require('path');
+const sharp = require('sharp');
const { updateFile } = require('~/models');
-const { prepareImagesLocal } = require('../images');
+const { prepareImagesLocal, uploadLocalImage } = require('../images');
describe('prepareImagesLocal', () => {
let tmpDir;
@@ -62,3 +58,46 @@ describe('prepareImagesLocal', () => {
expect(encoded).toBe(Buffer.from('plain-png-bytes').toString('base64'));
});
});
+
+describe('uploadLocalImage', () => {
+ let tmpDir;
+ let imageOutput;
+
+ beforeEach(() => {
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'upload-local-image-'));
+ imageOutput = path.join(tmpDir, 'images');
+ });
+
+ afterEach(() => {
+ fs.rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ it.each(['jpeg', 'png'])(
+ 'stores %s bytes named .png in the configured PNG format',
+ async (format) => {
+ const inputFilePath = path.join(tmpDir, 'photo.png');
+ await sharp({ create: { width: 8, height: 8, channels: 3, background: '#d22' } })
+ .toFormat(format)
+ .toFile(inputFilePath);
+
+ const result = await uploadLocalImage({
+ req: { user: { id: 'user-1' }, config: { paths: { imageOutput }, imageOutputType: 'png' } },
+ file: { path: inputFilePath },
+ file_id: 'file-1',
+ endpoint: 'anthropic',
+ });
+ const stored = await fs.promises.readFile(
+ path.join(imageOutput, 'user-1', 'file-1__photo.png'),
+ );
+
+ expect((await sharp(stored).metadata()).format).toBe('png');
+ expect(result).toEqual({
+ filepath: '/images/user-1/file-1__photo.png',
+ bytes: stored.length,
+ width: 8,
+ height: 8,
+ });
+ expect(fs.existsSync(inputFilePath)).toBe(false);
+ },
+ );
+});
diff --git a/api/server/services/Files/Local/images.js b/api/server/services/Files/Local/images.js
index ebfaf1e4c4..51eee41c20 100644
--- a/api/server/services/Files/Local/images.js
+++ b/api/server/services/Files/Local/images.js
@@ -1,7 +1,7 @@
const fs = require('fs');
const path = require('path');
const sharp = require('sharp');
-const { stripCacheBust } = require('@librechat/api');
+const { stripCacheBust, convertImageBuffer } = require('@librechat/api');
const { resizeImageBuffer } = require('../images/resize');
const { updateUser, updateFile } = require('~/models');
@@ -36,34 +36,22 @@ async function uploadLocalImage({ req, file, file_id, endpoint, resolution = 'hi
buffer: resizedBuffer,
width,
height,
+ type,
} = await resizeImageBuffer(inputBuffer, resolution, endpoint);
- const extension = path.extname(inputFilePath);
+ const { buffer: outputBuffer, fileName } = await convertImageBuffer({
+ buffer: resizedBuffer,
+ type,
+ fileName: `${file_id}__${path.basename(inputFilePath)}`,
+ outputType: appConfig.imageOutputType,
+ });
const { imageOutput } = appConfig.paths;
const userPath = path.join(imageOutput, req.user.id);
-
- if (!fs.existsSync(userPath)) {
- fs.mkdirSync(userPath, { recursive: true });
- }
-
- const fileName = `${file_id}__${path.basename(inputFilePath)}`;
- const newPath = path.join(userPath, fileName);
- const targetExtension = `.${appConfig.imageOutputType}`;
-
- if (extension.toLowerCase() === targetExtension) {
- const bytes = Buffer.byteLength(resizedBuffer);
- await fs.promises.writeFile(newPath, resizedBuffer);
- const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));
- return { filepath, bytes, width, height };
- }
-
- const outputFilePath = newPath.replace(extension, targetExtension);
- const data = await sharp(resizedBuffer).toFormat(appConfig.imageOutputType).toBuffer();
- await fs.promises.writeFile(outputFilePath, data);
- const bytes = Buffer.byteLength(data);
- const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));
+ await fs.promises.mkdir(userPath, { recursive: true });
+ await fs.promises.writeFile(path.join(userPath, fileName), outputBuffer);
await fs.promises.unlink(inputFilePath);
- return { filepath, bytes, width, height };
+ const filepath = path.posix.join('/', 'images', req.user.id, fileName);
+ return { filepath, bytes: outputBuffer.length, width, height };
}
/**
diff --git a/api/server/services/Files/images/convert.js b/api/server/services/Files/images/convert.js
index 09d8c0ecc3..2752d7e451 100644
--- a/api/server/services/Files/images/convert.js
+++ b/api/server/services/Files/images/convert.js
@@ -1,8 +1,7 @@
const fs = require('fs');
const path = require('path');
-const sharp = require('sharp');
const { logger } = require('@librechat/data-schemas');
-const { getStorageMetadata } = require('@librechat/api');
+const { getStorageMetadata, convertImageBuffer } = require('@librechat/api');
const { getStrategyFunctions } = require('../strategies');
const { resizeImageBuffer } = require('./resize');
@@ -20,8 +19,6 @@ async function convertImage(req, file, resolution = 'high', basename = '') {
try {
const appConfig = req.config;
let inputBuffer;
- let outputBuffer;
- let extension = path.extname(file.path ?? basename).toLowerCase();
// Check if the input is a buffer or a file path
if (Buffer.isBuffer(file)) {
@@ -38,20 +35,15 @@ async function convertImage(req, file, resolution = 'high', basename = '') {
buffer: resizedBuffer,
width,
height,
+ type,
} = await resizeImageBuffer(inputBuffer, resolution);
- // Check if the file is already in target format; if it isn't, convert it:
- const targetExtension = `.${appConfig.imageOutputType}`;
- if (extension === targetExtension) {
- outputBuffer = resizedBuffer;
- } else {
- outputBuffer = await sharp(resizedBuffer).toFormat(appConfig.imageOutputType).toBuffer();
- extension = targetExtension;
- }
-
- // Generate a new filename for the output file
- const newFileName =
- path.basename(file.path ?? basename, path.extname(file.path ?? basename)) + extension;
+ const { buffer: outputBuffer, fileName: newFileName } = await convertImageBuffer({
+ buffer: resizedBuffer,
+ type,
+ fileName: path.basename(file.path ?? basename),
+ outputType: appConfig.imageOutputType,
+ });
const { saveBuffer } = getStrategyFunctions(appConfig.fileStrategy);
diff --git a/packages/api/src/files/encode/image.spec.ts b/packages/api/src/files/encode/image.spec.ts
index f447faedc1..b3017a162b 100644
--- a/packages/api/src/files/encode/image.spec.ts
+++ b/packages/api/src/files/encode/image.spec.ts
@@ -1,3 +1,4 @@
+import sharp from 'sharp';
import { Readable } from 'node:stream';
import { FileSources, VisionModes } from 'librechat-data-provider';
import type { ServerRequest } from '~/types';
@@ -202,6 +203,110 @@ describe('encodeAndFormatImages', () => {
expect(result.image_urls).toEqual([expected]);
});
+ describe('a record whose type does not describe its bytes', () => {
+ let jpegBytes: Buffer;
+ let jpegContent: string;
+ const mislabeled = () => ({ ...file, bytes: jpegBytes.length, type: 'image/png' });
+
+ beforeAll(async () => {
+ jpegBytes = await sharp({
+ create: { width: 8, height: 8, channels: 3, background: { r: 1, g: 2, b: 3 } },
+ })
+ .jpeg()
+ .toBuffer();
+ jpegContent = jpegBytes.toString('base64');
+ });
+
+ beforeEach(() => {
+ getDownloadStream.mockImplementation(async () => Readable.from(jpegBytes));
+ });
+
+ it('declares the bytes to Anthropic, not the stored type', async () => {
+ const result = await encodeAndFormatImages(
+ makeReq(),
+ [mislabeled()],
+ { endpoint: 'anthropic' },
+ deps,
+ );
+
+ expect(result.image_urls).toEqual([
+ { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: jpegContent } },
+ ]);
+ });
+
+ it('uses the real type for Claude conversations in agents mode', async () => {
+ const result = await encodeAndFormatImages(
+ makeReq(),
+ [mislabeled()],
+ { provider: 'anthropic' },
+ deps,
+ VisionModes.agents,
+ );
+
+ expect(result.image_urls).toEqual([
+ {
+ type: 'image_url',
+ image_url: { url: `data:image/jpeg;base64,${jpegContent}`, detail: 'auto' },
+ },
+ ]);
+ });
+
+ it('declares the bytes to Google', async () => {
+ const result = await encodeAndFormatImages(
+ makeReq(),
+ [mislabeled()],
+ { provider: 'google' },
+ deps,
+ VisionModes.generative,
+ );
+
+ expect(result.image_urls).toEqual([
+ { type: 'image_url', inlineData: { mimeType: 'image/jpeg', data: jpegContent } },
+ ]);
+ });
+
+ it('declares the bytes in the data URL', async () => {
+ const result = await encodeAndFormatImages(makeReq(), [mislabeled()], {}, deps);
+
+ expect(result.image_urls).toEqual([
+ {
+ type: 'image_url',
+ image_url: { url: `data:image/jpeg;base64,${jpegContent}`, detail: 'auto' },
+ },
+ ]);
+ });
+
+ it('keeps the stored type when the bytes name no format of their own', async () => {
+ getDownloadStream.mockImplementation(async () => Readable.from(imageBytes));
+
+ const result = await encodeAndFormatImages(
+ makeReq(),
+ [{ ...file, type: 'image/png' }],
+ { endpoint: 'anthropic' },
+ deps,
+ );
+
+ expect(result.image_urls).toEqual([
+ { type: 'image', source: { type: 'base64', media_type: 'image/png', data: content } },
+ ]);
+ });
+
+ it('leaves an external URL alone, having no bytes to read', async () => {
+ prepareImagePayload.mockImplementation(async (_req, image) => [image, file.filepath]);
+
+ const result = await encodeAndFormatImages(
+ makeReq(),
+ [{ ...mislabeled(), source: FileSources.openai }],
+ { endpoint: 'openai' },
+ deps,
+ );
+
+ expect(result.image_urls).toEqual([
+ { type: 'image_url', image_url: { url: file.filepath, detail: 'auto' } },
+ ]);
+ });
+ });
+
it.each([
[undefined, undefined, 'auto'],
['low', undefined, 'low'],
diff --git a/packages/api/src/files/encode/image.ts b/packages/api/src/files/encode/image.ts
index bcd486a54d..3aae189831 100644
--- a/packages/api/src/files/encode/image.ts
+++ b/packages/api/src/files/encode/image.ts
@@ -13,6 +13,7 @@ import type { AxiosInstance } from 'axios';
import type { ServerRequest, StrategyFunctions } from '~/types';
import { getFileStream, isAttachmentObjectNotFoundError } from './utils';
import { validateImage } from '~/files/validation';
+import { sniffImageMimeType } from '~/files/mime';
import { runGuardedEncode } from './memoryGuard';
import { logAxiosError } from '~/utils/axios';
@@ -185,33 +186,38 @@ export async function encodeAndFormatImages(
}
const isURL = imageContent.startsWith('http');
- if (file.height && file.width && !isURL) {
+ // Historical uploads may have a stored type that disagrees with their bytes.
+ let mediaType = file.type;
+ if (!isURL) {
const imageBuffer = Buffer.from(imageContent, 'base64');
- const validation = await validateImage(
- imageBuffer,
- imageBuffer.length,
- effectiveEndpoint ?? '',
- configuredFileSizeLimit,
- );
- if (!validation.isValid) {
- throw new Error(`Image validation failed for ${file.filename}: ${validation.error}`);
+ mediaType = sniffImageMimeType(imageBuffer) ?? file.type;
+ if (file.height && file.width) {
+ const validation = await validateImage(
+ imageBuffer,
+ imageBuffer.length,
+ effectiveEndpoint ?? '',
+ configuredFileSizeLimit,
+ );
+ if (!validation.isValid) {
+ throw new Error(`Image validation failed for ${file.filename}: ${validation.error}`);
+ }
}
}
- const url = isURL ? imageContent : `data:${file.type};base64,${imageContent}`;
+ const url = isURL ? imageContent : `data:${mediaType};base64,${imageContent}`;
if (mode === VisionModes.agents) {
result.image_urls.push({ type: ContentTypes.IMAGE_URL, image_url: { url, detail } });
} else if (effectiveEndpoint === EModelEndpoint.google && mode === VisionModes.generative) {
result.image_urls.push({
type: ContentTypes.IMAGE_URL,
- inlineData: { mimeType: file.type, data: imageContent },
+ inlineData: { mimeType: mediaType, data: imageContent },
});
} else if (effectiveEndpoint === EModelEndpoint.google) {
result.image_urls.push({ type: ContentTypes.IMAGE_URL, image_url: url });
} else if (effectiveEndpoint === EModelEndpoint.anthropic) {
result.image_urls.push({
type: 'image',
- source: { type: 'base64', media_type: file.type, data: imageContent },
+ source: { type: 'base64', media_type: mediaType, data: imageContent },
});
} else {
result.image_urls.push({ type: ContentTypes.IMAGE_URL, image_url: { url, detail } });
diff --git a/packages/api/src/files/format.spec.ts b/packages/api/src/files/format.spec.ts
new file mode 100644
index 0000000000..155aff537a
--- /dev/null
+++ b/packages/api/src/files/format.spec.ts
@@ -0,0 +1,69 @@
+import sharp from 'sharp';
+import { convertImageBuffer } from './format';
+
+const jpeg = async (): Promise<Buffer> =>
+ sharp({ create: { width: 8, height: 8, channels: 3, background: '#d22' } })
+ .jpeg()
+ .toBuffer();
+
+describe('convertImageBuffer', () => {
+ it('converts JPEG bytes with a PNG filename into an actual PNG', async () => {
+ const image = await convertImageBuffer({
+ buffer: await jpeg(),
+ type: 'image/jpeg',
+ fileName: 'photo.png',
+ outputType: 'png',
+ });
+
+ expect(image.fileName).toBe('photo.png');
+ expect((await sharp(image.buffer).metadata()).format).toBe('png');
+ });
+
+ it('preserves the buffer when both its type and filename match', async () => {
+ const buffer = await jpeg();
+ const image = await convertImageBuffer({
+ buffer,
+ type: 'image/jpeg',
+ fileName: 'photo.JPEG',
+ outputType: 'jpeg',
+ });
+
+ expect(image).toEqual({ buffer, fileName: 'photo.JPEG' });
+ expect(image.buffer).toBe(buffer);
+ });
+
+ it('converts when the resized format is unknown', async () => {
+ const image = await convertImageBuffer({
+ buffer: await jpeg(),
+ fileName: 'photo.png',
+ outputType: 'png',
+ });
+
+ expect((await sharp(image.buffer).metadata()).format).toBe('png');
+ });
+
+ it.each(['photo.jpg', 'photo', 'photo.png.jpg'])(
+ 'normalizes the filename %s',
+ async (fileName) => {
+ const image = await convertImageBuffer({
+ buffer: await jpeg(),
+ type: 'image/jpeg',
+ fileName,
+ outputType: 'webp',
+ });
+
+ expect(image.fileName).toBe(fileName === 'photo.png.jpg' ? 'photo.png.webp' : 'photo.webp');
+ expect((await sharp(image.buffer).metadata()).format).toBe('webp');
+ },
+ );
+
+ it('propagates unexpected conversion failures', async () => {
+ await expect(
+ convertImageBuffer({
+ buffer: Buffer.from('invalid'),
+ fileName: 'photo.png',
+ outputType: 'png',
+ }),
+ ).rejects.toThrow();
+ });
+});
diff --git a/packages/api/src/files/format.ts b/packages/api/src/files/format.ts
new file mode 100644
index 0000000000..baae9d0191
--- /dev/null
+++ b/packages/api/src/files/format.ts
@@ -0,0 +1,31 @@
+import path from 'path';
+import sharp from 'sharp';
+import type { FormatEnum } from 'sharp';
+
+type ImageFormatInput = {
+ buffer: Buffer;
+ type?: string;
+ fileName: string;
+ outputType: string;
+};
+
+/** Normalizes resized bytes and their filename to the configured output format. */
+export async function convertImageBuffer({
+ buffer,
+ type,
+ fileName,
+ outputType,
+}: ImageFormatInput): Promise<{ buffer: Buffer; fileName: string }> {
+ const extension = path.extname(fileName);
+ const targetExtension = `.${outputType}`;
+ if (extension.toLowerCase() === targetExtension && type === `image/${outputType}`) {
+ return { buffer, fileName };
+ }
+
+ return {
+ buffer: await sharp(buffer)
+ .toFormat(outputType as keyof FormatEnum)
+ .toBuffer(),
+ fileName: `${path.basename(fileName, extension)}${targetExtension}`,
+ };
+}
diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.ts
index b08ebd4666..6843fb56c6 100644
--- a/packages/api/src/files/index.ts
+++ b/packages/api/src/files/index.ts
@@ -9,6 +9,7 @@ export * from './extract';
export * from './documents/crud';
export * from './encode';
export * from './filter';
+export * from './format';
export * from './mime';
export * from './mistral/crud';
export * from './ocr';
diff --git a/packages/api/src/files/mime.spec.ts b/packages/api/src/files/mime.spec.ts
index 9f28fb170b..d7dbb8d26f 100644
--- a/packages/api/src/files/mime.spec.ts
+++ b/packages/api/src/files/mime.spec.ts
@@ -1,14 +1,14 @@
import sharp from 'sharp';
-import { resolveImageMimeType } from './mime';
+import { resolveImageMimeType, sniffImageMimeType } from './mime';
-describe('resolveImageMimeType', () => {
- const encode = async (format: 'png' | 'jpeg' | 'webp' | 'gif' | 'tiff'): Promise<Buffer> =>
- await sharp({
- create: { width: 8, height: 8, channels: 4, background: { r: 1, g: 2, b: 3, alpha: 1 } },
- })
- [format]()
- .toBuffer();
+const encode = async (format: 'png' | 'jpeg' | 'webp' | 'gif' | 'tiff' | 'avif'): Promise<Buffer> =>
+ await sharp({
+ create: { width: 8, height: 8, channels: 4, background: { r: 1, g: 2, b: 3, alpha: 1 } },
+ })
+ [format]()
+ .toBuffer();
+describe('resolveImageMimeType', () => {
it.each([
['png', 'image/png'],
['jpeg', 'image/jpeg'],
@@ -46,3 +46,75 @@ describe('resolveImageMimeType', () => {
expect(resolveImageMimeType({ format: 'heif', compression: undefined })).toBeUndefined();
});
});
+
+describe('sniffImageMimeType', () => {
+ const ftyp = (major: string, ...compatible: string[]): Buffer => {
+ const buffer = Buffer.alloc(16 + compatible.length * 4);
+ buffer.writeUInt32BE(buffer.length, 0);
+ buffer.write('ftyp', 4, 'latin1');
+ buffer.write(major, 8, 'latin1');
+ compatible.forEach((brand, index) => buffer.write(brand, 16 + index * 4, 'latin1'));
+ return buffer;
+ };
+
+ it.each([
+ ['png', 'image/png'],
+ ['jpeg', 'image/jpeg'],
+ ['webp', 'image/webp'],
+ ['gif', 'image/gif'],
+ ['tiff', 'image/tiff'],
+ ] as const)('identifies real %s bytes as %s', async (format, expected) => {
+ const bytes = await encode(format);
+ expect(sniffImageMimeType(bytes)).toBe(expected);
+ expect(sniffImageMimeType(bytes)).toBe(resolveImageMimeType(await sharp(bytes).metadata()));
+ });
+
+ it.each(['avif', 'avis'])('identifies the explicit AVIF brand %s', (brand) => {
+ expect(sniffImageMimeType(ftyp(brand))).toBe('image/avif');
+ });
+
+ it.each(['heic', 'heix', 'hevc'])('identifies the explicit HEIC brand %s', (brand) => {
+ expect(sniffImageMimeType(ftyp(brand))).toBe('image/heic');
+ });
+
+ it.each(['mif1', 'msf1'])('resolves AVIF compatibility behind the generic %s brand', (brand) => {
+ expect(sniffImageMimeType(ftyp(brand, 'mif1', 'avif', 'miaf'))).toBe('image/avif');
+ });
+
+ it('identifies an encoded AVIF whose major brand is the generic HEIF brand', async () => {
+ const buffer = await encode('avif');
+ expect(sniffImageMimeType(buffer)).toBe('image/avif');
+ const pixels = await sharp(buffer).raw().toBuffer();
+ buffer.write('mif1', 8, 'latin1');
+ expect(sniffImageMimeType(buffer)).toBe('image/avif');
+ expect(await sharp(buffer).raw().toBuffer()).toEqual(pixels);
+ });
+
+ it('resolves HEIC compatibility behind a generic HEIF brand', () => {
+ expect(sniffImageMimeType(ftyp('mif1', 'mif1', 'heic'))).toBe('image/heic');
+ });
+
+ it('does not guess a codec from generic HEIF brands', () => {
+ expect(sniffImageMimeType(ftyp('mif1', 'mif1'))).toBeUndefined();
+ expect(sniffImageMimeType(ftyp('msf1'))).toBeUndefined();
+ expect(sniffImageMimeType(ftyp('isom', 'mp41'))).toBeUndefined();
+ });
+
+ it('does not read compatible brands past a truncated or undersized ftyp box', () => {
+ const truncated = ftyp('mif1', 'avif');
+ truncated.writeUInt32BE(truncated.length + 4, 0);
+ const undersized = ftyp('mif1', 'avif');
+ undersized.writeUInt32BE(12, 0);
+ const trailing = Buffer.concat([ftyp('mif1'), Buffer.from('avif')]);
+ expect(sniffImageMimeType(truncated)).toBeUndefined();
+ expect(sniffImageMimeType(undersized)).toBeUndefined();
+ expect(sniffImageMimeType(trailing)).toBeUndefined();
+ });
+
+ it('preserves the stored-type fallback for unknown, short, or non-image bytes', () => {
+ expect(sniffImageMimeType(Buffer.from('not an image at all'))).toBeUndefined();
+ expect(sniffImageMimeType(Buffer.alloc(0))).toBeUndefined();
+ expect(sniffImageMimeType(Buffer.from([0x89, 0x50, 0x4e, 0x47]))).toBeUndefined();
+ expect(sniffImageMimeType(Buffer.from('RIFF\0\0\0\0WAVE', 'latin1'))).toBeUndefined();
+ });
+});
diff --git a/packages/api/src/files/mime.ts b/packages/api/src/files/mime.ts
index 1fa85a46d0..c20ea13790 100644
--- a/packages/api/src/files/mime.ts
+++ b/packages/api/src/files/mime.ts
@@ -44,3 +44,56 @@ export function resolveImageMimeType(metadata: EncodedImageMetadata): string | u
}
return SHARP_FORMAT_MIME_TYPES[format];
}
+
+const ISO_BMFF_BRAND_MIME_TYPES: Readonly<Record<string, string>> = {
+ avif: 'image/avif',
+ avis: 'image/avif',
+ heic: 'image/heic',
+ heim: 'image/heic',
+ heis: 'image/heic',
+ heix: 'image/heic',
+ hevc: 'image/heic',
+ hevx: 'image/heic',
+};
+
+/** Reads image signatures without decoding; unknown bytes retain the caller's stored type. */
+export function sniffImageMimeType(buffer: Buffer): string | undefined {
+ if (buffer.length < 12) {
+ return undefined;
+ }
+ if (buffer.readUInt32BE(0) === 0x89504e47 && buffer.readUInt32BE(4) === 0x0d0a1a0a) {
+ return 'image/png';
+ }
+ if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+ return 'image/jpeg';
+ }
+ const leading = buffer.toString('latin1', 0, 12);
+ if (leading.startsWith('GIF87a') || leading.startsWith('GIF89a')) {
+ return 'image/gif';
+ }
+ if (leading.startsWith('RIFF') && leading.slice(8, 12) === 'WEBP') {
+ return 'image/webp';
+ }
+ if (leading.slice(4, 8) === 'ftyp') {
+ const majorType = ISO_BMFF_BRAND_MIME_TYPES[leading.slice(8, 12)];
+ if (majorType) {
+ return majorType;
+ }
+ // Generic HEIF brands (mif1/msf1) do not identify a codec. Check compatible brands.
+ const boxSize = buffer.readUInt32BE(0);
+ if (boxSize < 16 || boxSize > buffer.length) {
+ return undefined;
+ }
+ for (let offset = 16; offset + 4 <= boxSize; offset += 4) {
+ const mediaType = ISO_BMFF_BRAND_MIME_TYPES[buffer.toString('latin1', offset, offset + 4)];
+ if (mediaType) {
+ return mediaType;
+ }
+ }
+ return undefined;
+ }
+ if (leading.startsWith('II\x2a\x00') || leading.startsWith('MM\x00\x2a')) {
+ return 'image/tiff';
+ }
+ return undefined;
+}
diff --git a/packages/api/src/storage/__tests__/images.test.ts b/packages/api/src/storage/__tests__/images.test.ts
index ec8f71910e..e495e128f9 100644
--- a/packages/api/src/storage/__tests__/images.test.ts
+++ b/packages/api/src/storage/__tests__/images.test.ts
@@ -1,8 +1,8 @@
import fs from 'fs';
import sharp from 'sharp';
-import type { ServerRequest } from '~/types';
-import type { SaveBufferFn } from '~/storage/types';
import type { ImageServiceDeps } from '~/storage/images';
+import type { SaveBufferFn } from '~/storage/types';
+import type { ServerRequest } from '~/types';
import { ImageService } from '~/storage/images';
jest.mock('fs', () => {
@@ -43,6 +43,7 @@ describe('ImageService', () => {
buffer: Buffer.from('resized'),
width: 800,
height: 600,
+ type: 'image/jpeg',
}),
updateUser: jest.fn().mockResolvedValue(undefined),
updateFile: jest.fn().mockResolvedValue(null),
@@ -108,11 +109,18 @@ describe('ImageService', () => {
).rejects.toThrow('User not authenticated');
});
- it('skips format conversion when extension matches target', async () => {
- const webpFile = {
- path: '/tmp/upload-123.webp',
- originalname: 'photo.webp',
- } as Express.Multer.File;
+ const webpFile = {
+ path: '/tmp/upload-123.webp',
+ originalname: 'photo.webp',
+ } as Express.Multer.File;
+
+ it('skips format conversion when the name and the bytes both match the target', async () => {
+ (mockDeps.resizeImageBuffer as jest.Mock).mockResolvedValue({
+ buffer: Buffer.from('resized'),
+ width: 800,
+ height: 600,
+ type: 'image/webp',
+ });
await service.uploadImage({
req: mockReq as ServerRequest,
@@ -124,6 +132,35 @@ describe('ImageService', () => {
expect(sharp).not.toHaveBeenCalled();
});
+ it('converts bytes that are not in the target format, whatever the name says', async () => {
+ await service.uploadImage({
+ req: mockReq as ServerRequest,
+ file: webpFile,
+ file_id: 'file-456',
+ endpoint: 'openAI',
+ });
+
+ expect(sharp).toHaveBeenCalled();
+ });
+
+ it('converts when sharp named no format for the resized bytes', async () => {
+ (mockDeps.resizeImageBuffer as jest.Mock).mockResolvedValue({
+ buffer: Buffer.from('resized'),
+ width: 800,
+ height: 600,
+ type: undefined,
+ });
+
+ await service.uploadImage({
+ req: mockReq as ServerRequest,
+ file: webpFile,
+ file_id: 'file-456',
+ endpoint: 'openAI',
+ });
+
+ expect(sharp).toHaveBeenCalled();
+ });
+
it('uses custom resolution when provided', async () => {
await service.uploadImage({
req: mockReq as ServerRequest,
diff --git a/packages/api/src/storage/images.ts b/packages/api/src/storage/images.ts
index e1c770923c..3b6b41e45b 100644
--- a/packages/api/src/storage/images.ts
+++ b/packages/api/src/storage/images.ts
@@ -4,7 +4,6 @@ import sharp from 'sharp';
import { logger } from '@librechat/data-schemas';
import type { IUser } from '@librechat/data-schemas';
import type { TFile } from 'librechat-data-provider';
-import type { FormatEnum } from 'sharp';
import type {
SaveBufferFn,
UploadImageParams,
@@ -12,13 +11,14 @@ import type {
ProcessAvatarParams,
} from '~/storage/types';
import { AVATAR_BASE_PATH, DEFAULT_BASE_PATH as defaultBasePath } from '~/storage/constants';
+import { convertImageBuffer } from '~/files/format';
export interface ImageServiceDeps {
resizeImageBuffer: (
buffer: Buffer,
resolution: string,
endpoint: string,
- ) => Promise<{ buffer: Buffer; width: number; height: number }>;
+ ) => Promise<{ buffer: Buffer; width: number; height: number; type?: string }>;
updateUser: (userId: string, update: { avatar: string }) => Promise<IUser | null>;
updateFile: (params: { file_id: string }) => Promise<TFile>;
}
@@ -71,26 +71,16 @@ export class ImageService {
buffer: resizedBuffer,
width,
height,
+ type,
} = await this.deps.resizeImageBuffer(inputBuffer, resolution, endpoint);
- const extension = path.extname(inputFilePath);
const userId = req.user.id;
- const outputType = appConfig?.imageOutputType ?? 'webp';
- const targetExtension = `.${outputType}`;
-
- let processedBuffer: Buffer;
- let fileName = `${file_id}__${path.basename(inputFilePath)}`;
-
- if (extension.toLowerCase() === targetExtension) {
- processedBuffer = resizedBuffer;
- } else {
- const outputFormat = outputType as keyof FormatEnum;
- processedBuffer = await sharp(resizedBuffer).toFormat(outputFormat).toBuffer();
- fileName = fileName.replace(new RegExp(path.extname(fileName) + '$'), targetExtension);
- if (!path.extname(fileName)) {
- fileName += targetExtension;
- }
- }
+ const { buffer: processedBuffer, fileName } = await convertImageBuffer({
+ buffer: resizedBuffer,
+ type,
+ fileName: `${file_id}__${path.basename(inputFilePath)}`,
+ outputType: appConfig?.imageOutputType ?? 'webp',
+ });
const downloadURL = await this.saveBuffer({
userId,