Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🖍️ fix: Honor Agent Image Detail Settings (#15407)

Danny Avila413906abc372
diff --git a/api/server/controllers/agents/client.imageDetail.spec.js b/api/server/controllers/agents/client.imageDetail.spec.js
new file mode 100644
index 0000000000..db49a84bc0
--- /dev/null
+++ b/api/server/controllers/agents/client.imageDetail.spec.js
@@ -0,0 +1,55 @@
+const mockEncodeAndFormat = jest.fn().mockResolvedValue({ files: [], image_urls: [] });
+
+jest.mock('~/server/services/Files/images/encode', () => ({
+ encodeAndFormat: (...args) => mockEncodeAndFormat(...args),
+}));
+
+const AgentClient = require('./client');
+
+describe('AgentClient.addImageURLs - image detail', () => {
+ const buildClient = (options) => {
+ const client = Object.create(AgentClient.prototype);
+ client.options = {
+ req: { body: {} },
+ endpoint: 'agents',
+ agent: { id: 'agent_1', provider: 'openai' },
+ ...options,
+ };
+ return client;
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it("forwards the agent's configured detail to the encoder", async () => {
+ const client = buildClient({ imageDetail: 'high' });
+
+ await client.addImageURLs({}, [{ file_id: 'f_1' }]);
+
+ expect(mockEncodeAndFormat).toHaveBeenCalledTimes(1);
+ expect(mockEncodeAndFormat.mock.calls[0][2]).toEqual(
+ expect.objectContaining({ imageDetail: 'high' }),
+ );
+ });
+
+ it('leaves the detail undefined when the agent configures none, so the encoder falls back', async () => {
+ const client = buildClient({});
+
+ await client.addImageURLs({}, [{ file_id: 'f_1' }]);
+
+ expect(mockEncodeAndFormat.mock.calls[0][2].imageDetail).toBeUndefined();
+ });
+
+ it('still passes the provider and endpoint alongside the detail', async () => {
+ const client = buildClient({ imageDetail: 'low' });
+
+ await client.addImageURLs({}, [{ file_id: 'f_1' }]);
+
+ expect(mockEncodeAndFormat.mock.calls[0][2]).toEqual({
+ provider: 'openai',
+ endpoint: 'agents',
+ imageDetail: 'low',
+ });
+ });
+});
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index 3b12941947..9a90f46a1a 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -1675,6 +1675,7 @@ class AgentClient extends BaseClient {
{
provider: this.options.agent.provider,
endpoint: this.options.endpoint,
+ imageDetail: this.options.imageDetail,
},
VisionModes.agents,
);
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index 42c0f82ccb..53732e2e5d 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -1567,6 +1567,7 @@ const initializeClient = async ({
agentContextAttachmentsByAgentId,
endpointType: endpointOption.endpointType,
resendFiles: primaryConfig.resendFiles ?? true,
+ imageDetail: primaryConfig.imageDetail,
maxContextTokens: primaryConfig.maxContextTokens,
endpoint: isEphemeralAgentId(primaryConfig.id) ? primaryConfig.endpoint : EModelEndpoint.agents,
subagentAggregatorsByToolCallId,
diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js
index 25110566ee..dc744542f1 100644
--- a/api/server/services/Files/images/encode.js
+++ b/api/server/services/Files/images/encode.js
@@ -94,6 +94,8 @@ const blobStorageSources = new Set([
* @param {object} params - Object containing provider/endpoint information
* @param {Providers | EModelEndpoint | string} [params.provider] - The provider for the image
* @param {string} [params.endpoint] - Optional: The endpoint for the image
+ * @param {string} [params.imageDetail] - Optional: Detail level resolved by the caller, used
+ * where the request body carries no conversation-level setting (the agents route).
* @param {string} [mode] - Optional: The endpoint mode for the image.
* @returns {Promise<{ files: MongoFile[]; image_urls: MessageContentImageUrl[] }>} - A promise that resolves to the result object containing the encoded images and file details.
*/
@@ -159,7 +161,7 @@ async function encodeAndFormat(req, files, params, mode) {
promises.push(preparePayload(req, file));
}
- const detail = req.body.imageDetail ?? ImageDetail.auto;
+ const detail = params.imageDetail ?? req.body.imageDetail ?? ImageDetail.auto;
/** @type {Array<[MongoFile, string]>} */
const formattedImages = await Promise.all(promises);
diff --git a/api/server/services/Files/images/encode.spec.js b/api/server/services/Files/images/encode.spec.js
index 700f930d32..262a55be45 100644
--- a/api/server/services/Files/images/encode.spec.js
+++ b/api/server/services/Files/images/encode.spec.js
@@ -109,3 +109,56 @@ describe('encodeAndFormat - request memory guard', () => {
expect(result.image_urls[0].image_url.url).toBe(`data:image/png;base64,${localBase64}`);
});
});
+
+describe('encodeAndFormat - image detail', () => {
+ const imageFile = () => ({
+ source: FileSources.s3,
+ height: 10,
+ width: 10,
+ type: 'image/png',
+ file_id: 'f-detail',
+ filepath: 'bucket/a.png',
+ filename: 'a.png',
+ bytes: 128,
+ });
+
+ beforeEach(() => {
+ mockGetDownloadStream.mockResolvedValue(Readable.from([Buffer.from('image-bytes')]));
+ });
+
+ it('falls back to auto when no detail is configured anywhere', async () => {
+ const result = await encodeAndFormat(makeReq(), [imageFile()], { endpoint: 'openai' });
+
+ expect(result.image_urls[0].image_url.detail).toBe('auto');
+ });
+
+ it('uses the conversation-level detail from the request body', async () => {
+ const req = makeReq();
+ req.body.imageDetail = 'low';
+
+ const result = await encodeAndFormat(req, [imageFile()], { endpoint: 'openai' });
+
+ expect(result.image_urls[0].image_url.detail).toBe('low');
+ });
+
+ it('uses the detail resolved by the caller on routes with no body setting', async () => {
+ const result = await encodeAndFormat(makeReq(), [imageFile()], {
+ endpoint: 'agents',
+ imageDetail: 'high',
+ });
+
+ expect(result.image_urls[0].image_url.detail).toBe('high');
+ });
+
+ it('prefers the caller-resolved detail over the request body', async () => {
+ const req = makeReq();
+ req.body.imageDetail = 'low';
+
+ const result = await encodeAndFormat(req, [imageFile()], {
+ endpoint: 'agents',
+ imageDetail: 'high',
+ });
+
+ expect(result.image_urls[0].image_url.detail).toBe('high');
+ });
+});
diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts
index 0dba003e2b..097aef6f32 100644
--- a/packages/api/src/agents/initialize.ts
+++ b/packages/api/src/agents/initialize.ts
@@ -20,10 +20,11 @@ import type {
AgentToolOptions,
TEndpointOption,
ReasoningResponseKey,
+ StatefulCodeEnvironment,
+ ImageDetail,
TFile,
Agent,
TUser,
- StatefulCodeEnvironment,
} from 'librechat-data-provider';
import type { GenericTool, LCToolRegistry, ToolMap, LCTool } from '@librechat/agents';
import type { IMongoFile, FileOwnerScope } from '@librechat/data-schemas';
@@ -373,6 +374,9 @@ export type InitializedAgent = Agent & {
baseContextTokens?: number;
useLegacyContent: boolean;
resendFiles: boolean;
+ /** Detail level LibreChat encodes image content blocks with, from the agent's
+ * model parameters. Absent when the agent does not configure one. */
+ imageDetail?: ImageDetail;
tool_resources?: AgentToolResources;
userMCPAuthMap?: Record<string, Record<string, string>>;
/** Tool map for ToolNode to use when executing tools (required for PTC) */
@@ -950,7 +954,7 @@ export async function initializeAgent(
),
);
- const { resendFiles, maxContextTokens, modelOptions } = extractLibreChatParams(
+ const { resendFiles, maxContextTokens, imageDetail, modelOptions } = extractLibreChatParams(
_modelOptions as Record<string, unknown>,
);
@@ -1776,6 +1780,7 @@ export async function initializeAgent(
const initializedAgent: InitializedAgent = {
...agent,
resendFiles,
+ imageDetail,
toolRegistry,
mcpAvailableTools,
requestScopedConnections,
diff --git a/packages/api/src/utils/llm.test.ts b/packages/api/src/utils/llm.test.ts
index c4d270f400..3c652d4cce 100644
--- a/packages/api/src/utils/llm.test.ts
+++ b/packages/api/src/utils/llm.test.ts
@@ -1,3 +1,4 @@
+import { ImageDetail } from 'librechat-data-provider';
import { extractLibreChatParams } from './llm';
describe('extractLibreChatParams', () => {
@@ -30,6 +31,7 @@ describe('extractLibreChatParams', () => {
maxContextTokens: 4096,
fileTokenLimit: 50000,
modelLabel: 'GPT-4',
+ imageDetail: ImageDetail.high,
model: 'gpt-4',
temperature: 0.7,
max_tokens: 1000,
@@ -42,6 +44,7 @@ describe('extractLibreChatParams', () => {
expect(result.maxContextTokens).toBe(4096);
expect(result.fileTokenLimit).toBe(50000);
expect(result.modelLabel).toBe('GPT-4');
+ expect(result.imageDetail).toBe(ImageDetail.high);
expect(result.modelOptions).toEqual({
model: 'gpt-4',
temperature: 0.7,
@@ -49,6 +52,21 @@ describe('extractLibreChatParams', () => {
});
});
+ it('should keep imageDetail out of the model options sent to the provider', () => {
+ const result = extractLibreChatParams({ model: 'gpt-4o', imageDetail: ImageDetail.low });
+
+ expect(result.imageDetail).toBe(ImageDetail.low);
+ expect(result.modelOptions).toEqual({ model: 'gpt-4o' });
+ expect('imageDetail' in result.modelOptions).toBe(false);
+ });
+
+ it('should leave imageDetail undefined when the agent configures none', () => {
+ const result = extractLibreChatParams({ model: 'gpt-4o' });
+
+ expect(result.imageDetail).toBeUndefined();
+ expect(result.modelOptions).toEqual({ model: 'gpt-4o' });
+ });
+
it('should handle null values for LibreChat params', () => {
const options = {
resendFiles: true,
diff --git a/packages/api/src/utils/llm.ts b/packages/api/src/utils/llm.ts
index d1e1d9a43b..bd2eb73ba2 100644
--- a/packages/api/src/utils/llm.ts
+++ b/packages/api/src/utils/llm.ts
@@ -1,5 +1,5 @@
import { librechat } from 'librechat-data-provider';
-import type { DynamicSettingProps } from 'librechat-data-provider';
+import type { DynamicSettingProps, ImageDetail } from 'librechat-data-provider';
type LibreChatKeys = keyof typeof librechat;
@@ -10,6 +10,7 @@ type LibreChatParams = {
maxContextTokens?: number;
fileTokenLimit?: number;
modelLabel?: string | null;
+ imageDetail?: ImageDetail;
};
/**
@@ -35,6 +36,7 @@ export function extractLibreChatParams(
const maxContextTokens = (delete modelOptions.maxContextTokens, options.maxContextTokens);
const fileTokenLimit = (delete modelOptions.fileTokenLimit, options.fileTokenLimit);
const modelLabel = (delete modelOptions.modelLabel, options.modelLabel);
+ const imageDetail = (delete modelOptions.imageDetail, options.imageDetail);
return {
modelOptions: modelOptions as Omit<
@@ -46,5 +48,6 @@ export function extractLibreChatParams(
promptPrefix,
resendFiles,
modelLabel,
+ imageDetail,
};
}
diff --git a/packages/data-provider/src/parameterSettings.ts b/packages/data-provider/src/parameterSettings.ts
index 1a6a2cad66..e6898c1074 100644
--- a/packages/data-provider/src/parameterSettings.ts
+++ b/packages/data-provider/src/parameterSettings.ts
@@ -71,24 +71,6 @@ const baseDefinitions: Record<string, SettingDefinition> = {
minTags: 0,
maxTags: 4,
},
- imageDetail: {
- key: 'imageDetail',
- label: 'com_endpoint_plug_image_detail',
- labelCode: true,
- description: 'com_endpoint_openai_detail',
- descriptionCode: true,
- type: 'enum',
- default: ImageDetail.auto,
- component: 'slider',
- options: [ImageDetail.low, ImageDetail.auto, ImageDetail.high],
- enumMappings: {
- [ImageDetail.low]: 'com_ui_low',
- [ImageDetail.auto]: 'com_ui_auto',
- [ImageDetail.high]: 'com_ui_high',
- },
- optionType: 'conversation',
- columnSpan: 2,
- },
};
const createDefinition = (
@@ -147,6 +129,26 @@ export const librechat = {
placeholderCode: true,
optionType: 'model',
} as const,
+ /** Controls how LibreChat encodes image content blocks, not a provider request
+ * parameter — so it belongs to this group and is stripped from model options. */
+ imageDetail: {
+ key: 'imageDetail',
+ label: 'com_endpoint_plug_image_detail',
+ labelCode: true,
+ description: 'com_endpoint_openai_detail',
+ descriptionCode: true,
+ type: 'enum',
+ default: ImageDetail.auto,
+ component: 'slider',
+ options: [ImageDetail.low, ImageDetail.auto, ImageDetail.high],
+ enumMappings: {
+ [ImageDetail.low]: 'com_ui_low',
+ [ImageDetail.auto]: 'com_ui_auto',
+ [ImageDetail.high]: 'com_ui_high',
+ },
+ optionType: 'conversation',
+ columnSpan: 2,
+ } as SettingDefinition,
fileTokenLimit: {
key: 'fileTokenLimit',
label: 'com_ui_file_token_limit',
@@ -887,7 +889,7 @@ const openAI: SettingsConfiguration = [
openAIParams.presence_penalty,
baseDefinitions.stop,
librechat.resendFiles,
- baseDefinitions.imageDetail,
+ librechat.imageDetail,
openAIParams.web_search,
openAIParams.reasoning_effort,
openAIParams.useResponsesApi,
@@ -920,7 +922,7 @@ const openAICol2: SettingsConfiguration = [
openAIParams.presence_penalty,
baseDefinitions.stop,
librechat.resendFiles,
- baseDefinitions.imageDetail,
+ librechat.imageDetail,
openAIParams.reasoning_effort,
openAIParams.reasoning_summary,
openAIParams.reasoning_mode,