Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import { FileContext, FileSources, ImageDetail } from 'librechat-data-provider';
import type { TFile } from 'librechat-data-provider';
import type { RunFileEncodingAgent, RunFileMessageEncoderDeps } from './encode';
import type { ServerRequest } from '~/types';
import { AgentAttachmentLimitError, AgentAttachmentPolicyError } from '../attachments';
import { resolveTurnDeliveryRouting } from './delivery';
import { createRunFileMessageEncoder } from './encode';
jest.mock('~/utils/tokenizer', () => ({ countTokens: (text: string) => text.length }));
import { extractFileContext } from '~/files/context';
const pdf: TFile = {
file_id: 'input-pdf',
filename: 'report.pdf',
type: 'application/pdf',
bytes: 20,
user: 'user',
embedded: false,
filepath: '/files/report.pdf',
object: 'file',
usage: 0,
source: FileSources.local,
llmDeliveryPath: 'provider',
text: 'Previously extracted text',
};
const nativeDocument = {
type: 'file',
file: { filename: 'report.pdf', file_data: 'data:application/pdf;base64,cGRm' },
};
const nativeImage = { type: 'image_url', image_url: { url: 'data:image/png;base64,aW1n' } };
/** A child as the host loads it, before initialization settles its delivery routing. */
type LoadedAgent = Omit<RunFileEncodingAgent, 'deliveryRouting'> & {
endpoint?: string;
useResponsesApi?: boolean;
};
function setup({
fileConfig = {},
agents = { child: { provider: 'openAI' } },
}: {
fileConfig?: NonNullable<ServerRequest['config']>['fileConfig'];
agents?: Record<string, LoadedAgent>;
} = {}) {
const req = { body: {}, config: { fileConfig } } as ServerRequest;
const initialized = Object.fromEntries(
Object.entries(agents).map(([id, { endpoint, useResponsesApi, ...agent }]) => [
id,
{
...agent,
deliveryRouting: resolveTurnDeliveryRouting({
agent: { provider: agent.provider, endpoint, model_parameters: { useResponsesApi } },
config: req.config,
}),
},
]),
);
const encodeImages = jest.fn(async () => ({ image_urls: [nativeImage] }));
const encodeDocuments = jest.fn(async () => ({ documents: [nativeDocument] }));
const encodeAudios = jest.fn(async () => ({ audios: [{ type: 'media', data: 'audio' }] }));
const encodeVideos = jest.fn(async () => ({ videos: [{ type: 'media', data: 'video' }] }));
const extractText = jest.fn<
ReturnType<RunFileMessageEncoderDeps['extractText']>,
Parameters<RunFileMessageEncoderDeps['extractText']>
>(extractFileContext);
const deps: RunFileMessageEncoderDeps = {
req,
getAgent: (id) => initialized[id],
encodeImages,
encodeDocuments,
encodeAudios,
encodeVideos,
getStrategyFunctions: jest.fn(),
extractText,
};
return { ...deps, ...createRunFileMessageEncoder(deps) };
}
describe('createRunFileMessageEncoder', () => {
it('sends PDF and image provider blocks to the child without duplicating extracted text', async () => {
const harness = setup({
agents: {
child: {
provider: 'openAI',
endpoint: 'child-provider',
model: 'child-model',
useResponsesApi: true,
imageDetail: ImageDetail.high,
},
},
});
const image: TFile = {
...pdf,
file_id: 'image',
filename: 'figure.png',
type: 'image/png',
embedded: true,
metadata: { fileIdentifier: 'already-provisioned' },
};
const messages = await harness.encode([pdf, image], 'child');
expect(messages).toHaveLength(1);
expect(messages[0]._getType()).toBe('human');
expect(messages[0].content).toEqual(expect.arrayContaining([nativeDocument, nativeImage]));
expect(JSON.stringify(messages[0].content)).not.toContain(pdf.text);
expect(harness.extractText).not.toHaveBeenCalled();
expect(harness.encodeDocuments).toHaveBeenCalledWith(
harness.req,
[pdf],
expect.objectContaining({
provider: 'openAI',
endpoint: 'child-provider',
model: 'child-model',
useResponsesApi: true,
imageDetail: 'high',
}),
harness.getStrategyFunctions,
);
expect(harness.encodeImages).toHaveBeenCalledWith(
harness.req,
[image],
expect.any(Object),
harness.getStrategyFunctions,
);
});
it('resolves one inferred input independently for provider, text and tool-only children', async () => {
const harness = setup({
agents: {
native: { provider: 'openAI', endpoint: 'native' },
text: { provider: 'openAI', endpoint: 'text' },
tools: { provider: 'openAI', endpoint: 'tools' },
},
fileConfig: {
endpoints: {
native: { defaultLLMDeliveryPath: { fallback: 'provider' } },
text: { defaultLLMDeliveryPath: { fallback: 'text' } },
tools: { defaultLLMDeliveryPath: { fallback: 'none' } },
},
},
});
const [native, text, tools] = await Promise.all([
harness.encode([pdf], 'native'),
harness.encode([pdf], 'text'),
harness.encode([pdf], 'tools'),
]);
expect(native[0].content).toEqual(expect.arrayContaining([nativeDocument]));
expect(JSON.stringify(text[0].content)).toContain(pdf.text);
expect(JSON.stringify(text[0].content)).not.toContain('data:application/pdf');
expect(tools).toEqual([]);
expect(harness.encodeDocuments).toHaveBeenCalledTimes(1);
expect(harness.extractText).toHaveBeenCalledWith(
expect.objectContaining({ attachments: [{ ...pdf, llmDeliveryPath: 'text' }] }),
);
expect(pdf.llmDeliveryPath).toBe('provider');
});
it('delivers stored text only to a child that runs no tool able to read a tool-routed file', async () => {
const csv: TFile = {
...pdf,
file_id: 'input-csv',
filename: 'sales.csv',
filepath: '/files/sales.csv',
type: 'text/csv',
text: 'region,total\nwest,4',
llmDeliveryPath: 'none',
metadata: { destinationChosen: false },
};
const harness = setup({
agents: {
noReader: { provider: 'openAI', fileConsumers: { executeCode: false, fileSearch: false } },
runsCode: { provider: 'openAI', fileConsumers: { executeCode: true, fileSearch: false } },
searches: { provider: 'openAI', fileConsumers: { executeCode: false, fileSearch: true } },
unknown: { provider: 'openAI' },
},
fileConfig: {
endpoints: {
openAI: {
defaultLLMDeliveryPath: { overrides: { 'text/csv': 'none' } },
textFallbackWithoutTools: true,
},
},
},
});
/* Run Code uploads the file on its first call, so the child that runs code keeps it off the
* prompt whether or not its sandbox holds a copy yet. File search serves only what its
* store holds, so the searching child receives the text for a file never embedded. */
const sandboxCsv: TFile = {
...csv,
metadata: {
destinationChosen: false,
codeEnvRef: {
kind: 'user',
id: 'user-1',
storage_session_id: 'session-1',
file_id: 'sandbox-input-csv',
},
},
};
const [noReader, runsCode, unprovisioned, searches, unknown] = await Promise.all([
harness.encode([csv], 'noReader'),
harness.encode([sandboxCsv], 'runsCode'),
harness.encode([csv], 'runsCode'),
harness.encode([csv], 'searches'),
harness.encode([csv], 'unknown'),
]);
expect(JSON.stringify(noReader[0].content)).toContain('region,total');
expect(runsCode).toEqual([]);
expect(unprovisioned).toEqual([]);
expect(JSON.stringify(searches[0].content)).toContain('region,total');
expect(unknown).toEqual([]);
expect(harness.extractText).toHaveBeenCalledWith(
expect.objectContaining({ attachments: [{ ...csv, llmDeliveryPath: 'text' }] }),
);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(csv.llmDeliveryPath).toBe('none');
});
it('honors an explicit tool destination even when the receiving endpoint supports native files', async () => {
const harness = setup();
const file: TFile = {
...pdf,
source: FileSources.text,
llmDeliveryPath: 'none',
metadata: { destinationChosen: true },
};
await expect(harness.encode([file], 'child')).resolves.toEqual([]);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(harness.extractText).not.toHaveBeenCalled();
});
it.each([
{
route: 'explicit tool destination',
file: { llmDeliveryPath: 'none', metadata: { destinationChosen: true } },
},
{
route: 'explicit tool destination on a text source',
file: {
source: FileSources.text,
llmDeliveryPath: 'none',
metadata: { destinationChosen: true },
},
},
{ route: 'inferred tool destination', file: { llmDeliveryPath: 'provider' } },
] satisfies { route: string; file: Partial<TFile> }[])(
'excludes $route files from model count, bytes and text budgets',
async ({ file }) => {
const harness = setup({
fileConfig: {
fileContextSizeLimit: 1,
fileContextCharLimit: 5,
endpoints: {
openAI: { fileLimit: 1, defaultLLMDeliveryPath: { fallback: 'none' } },
},
},
});
const input: TFile = { ...pdf, ...file, bytes: 2 * 1024 * 1024 };
await expect(
harness.encode([input, { ...input, file_id: 'second' }], 'child'),
).resolves.toEqual([]);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(harness.extractText).not.toHaveBeenCalled();
},
);
it.each(['provider', 'text'] as const)(
'counts an inferred tool-only input when the child resolves it to %s delivery',
async (fallback) => {
const harness = setup({
fileConfig: {
fileContextSizeLimit: 1,
endpoints: { openAI: { defaultLLMDeliveryPath: { fallback } } },
},
});
await expect(
harness.encode([{ ...pdf, llmDeliveryPath: 'none', bytes: 2 * 1024 * 1024 }], 'child'),
).rejects.toMatchObject({ limitType: 'bytes' });
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(harness.extractText).not.toHaveBeenCalled();
},
);
it('rejects a child text route without extracted text before reading any files', async () => {
const harness = setup({
fileConfig: {
endpoints: { openAI: { defaultLLMDeliveryPath: { fallback: 'text' } } },
},
});
await expect(harness.encode([{ ...pdf, text: undefined }], 'child')).rejects.toThrow(
'requires extracted text for this agent',
);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(harness.extractText).not.toHaveBeenCalled();
});
it('keeps legacy extracted text while excluding legacy tool-provisioned native bytes', async () => {
const harness = setup({
fileConfig: {
fileContextSizeLimit: 1,
endpoints: { openAI: { fileLimit: 1 } },
},
});
const files: TFile[] = [
{ ...pdf, file_id: 'legacy-text', source: FileSources.text, llmDeliveryPath: undefined },
{
...pdf,
file_id: 'legacy-tool',
embedded: true,
llmDeliveryPath: undefined,
bytes: 2 * 1024 * 1024,
},
];
const messages = await harness.encode(files, 'child');
expect(JSON.stringify(messages[0].content)).toContain(pdf.text);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
});
it('uses the upload MIME type for policy after image conversion', async () => {
const harness = setup({
fileConfig: { endpoints: { openAI: { supportedMimeTypes: ['image/heic'] } } },
});
const image: TFile = {
...pdf,
type: 'image/webp',
metadata: { routingMimeType: 'image/heic' },
};
const messages = await harness.encode([image], 'child');
expect(messages[0].content).toEqual(expect.arrayContaining([nativeImage]));
});
it('passes audio and video to the receiving provider encoders', async () => {
const harness = setup({ agents: { child: { provider: 'google' } } });
await harness.encode(
[
{ ...pdf, file_id: 'audio', type: 'audio/mpeg' },
{ ...pdf, file_id: 'video', type: 'video/mp4' },
],
'child',
);
expect(harness.encodeAudios).toHaveBeenCalledTimes(1);
expect(harness.encodeVideos).toHaveBeenCalledTimes(1);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
});
it.each([{ disabled: true }, { supportedMimeTypes: ['image/png'] }, { fileSizeLimit: 0.000001 }])(
'rejects incompatible child policy before reading bytes: %j',
async (policy) => {
const harness = setup({ fileConfig: { endpoints: { openAI: policy } } });
await expect(harness.encode([pdf], 'child')).rejects.toBeInstanceOf(
AgentAttachmentPolicyError,
);
await expect(
harness.encode(
[{ ...pdf, llmDeliveryPath: 'none', metadata: { destinationChosen: true } }],
'child',
),
).rejects.toBeInstanceOf(AgentAttachmentPolicyError);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(harness.extractText).not.toHaveBeenCalled();
},
);
it('leaves code outputs with cleared sandbox references out of the child prompt and its count budget', async () => {
/* Priming clears a dead reference on the turn's copy so the output is re-provisioned; the
* output still belongs to the sandbox, so it neither reaches the prompt nor spends the
* one-file budget. */
const harness = setup({ fileConfig: { endpoints: { openAI: { fileLimit: 1 } } } });
const output = (file_id: string): TFile => ({
...pdf,
file_id,
filename: `${file_id}.png`,
type: 'image/png',
text: undefined,
llmDeliveryPath: undefined,
context: FileContext.execute_code,
metadata: {},
});
await expect(harness.encode([output('chart-a'), output('chart-b')], 'child')).resolves.toEqual(
[],
);
expect(harness.encodeImages).not.toHaveBeenCalled();
});
it('includes permanent child context in the count budget before encoding shared files', async () => {
const harness = setup({
agents: {
child: {
provider: 'openAI',
agentContextAttachments: [{ ...pdf, file_id: 'permanent' }],
},
},
fileConfig: { endpoints: { openAI: { fileLimit: 1 } } },
});
await expect(harness.encode([pdf], 'child')).rejects.toMatchObject({ limitType: 'count' });
expect(harness.encodeDocuments).not.toHaveBeenCalled();
});
it('budgets mixed inputs with permanent model context once and excludes permanent tool files', async () => {
const permanent: TFile = { ...pdf, file_id: 'permanent' };
const tool: TFile = {
...pdf,
file_id: 'tool',
llmDeliveryPath: 'none',
metadata: { destinationChosen: true },
bytes: 2 * 1024 * 1024,
};
const harness = setup({
agents: {
child: {
provider: 'openAI',
agentContextAttachments: [permanent, { ...tool, file_id: 'permanent-tool' }],
},
},
fileConfig: {
fileContextSizeLimit: 1,
endpoints: { openAI: { fileLimit: 2 } },
},
});
const messages = await harness.encode([permanent, pdf, tool], 'child');
expect(messages).toHaveLength(1);
expect(messages[0].content).toEqual(expect.arrayContaining([nativeDocument]));
expect(harness.encodeDocuments).toHaveBeenCalledWith(
harness.req,
[permanent, pdf],
expect.any(Object),
harness.getStrategyFunctions,
);
expect(harness.extractText).not.toHaveBeenCalled();
});
it('rejects aggregate file bytes before encoding', async () => {
const harness = setup({ fileConfig: { endpoints: { openAI: { totalSizeLimit: 1 } } } });
await expect(
harness.encode(
[
{ ...pdf, bytes: 600_000 },
{ ...pdf, file_id: 'second', bytes: 600_000 },
],
'child',
),
).rejects.toMatchObject({ limitType: 'bytes' });
expect(harness.encodeDocuments).not.toHaveBeenCalled();
});
it('enforces the extracted-text budget before formatting the child prompt', async () => {
const harness = setup({ fileConfig: { fileContextCharLimit: 5 } });
await expect(
harness.encode(
[{ ...pdf, llmDeliveryPath: 'text', metadata: { destinationChosen: true } }],
'child',
),
).rejects.toBeInstanceOf(AgentAttachmentLimitError);
expect(harness.extractText).not.toHaveBeenCalled();
});
it('propagates encoder failures rather than silently dropping a shared input', async () => {
const harness = setup();
const failure = new Error('The input is no longer in storage');
jest.mocked(harness.encodeDocuments).mockRejectedValueOnce(failure);
await expect(harness.encode([pdf], 'child')).rejects.toBe(failure);
});
it.each(['provider', 'none'] as const)(
'screens %s file content with the deployment policy before encoding',
async (llmDeliveryPath) => {
const harness = setup();
harness.req.config!.filters = {
files: {
pii: {
fields: ['name'],
starterPatterns: [],
customPatterns: [{ id: 'private', label: 'private value', regex: 'PRIVATE-[A-Z]+' }],
},
},
};
await expect(
harness.encode(
[
{
...pdf,
filename: 'PRIVATE-DOC.pdf',
llmDeliveryPath,
metadata: { destinationChosen: true },
},
],
'child',
),
).rejects.toThrow('Submitted content contains a private value');
expect(harness.encodeDocuments).not.toHaveBeenCalled();
},
);
it('returns no extra message or storage access when no files are shared', async () => {
const harness = setup();
await expect(harness.encode([], 'child')).resolves.toEqual([]);
expect(harness.encodeDocuments).not.toHaveBeenCalled();
expect(harness.extractText).not.toHaveBeenCalled();
});
});