Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import crypto from 'node:crypto';
import {
Tools,
DEFAULT_MCP_APP_PERSISTED_BYTES,
MCP_APP_MIME_TYPE,
isHtmlMediaType,
isMcpAppMimeType,
resolveMCPAppsPolicy,
resolveMCPUIResourceMimeType,
} from 'librechat-data-provider';
import type { UIResource, TMCPAppsPolicy } from 'librechat-data-provider';
import type * as t from './types';
export const DEFAULT_MCP_IMAGE_DATA_MAX_BYTES: number = 10 * 1024 * 1024;
function generateResourceId(text: string): string {
return crypto.createHash('sha256').update(text).digest('hex').substring(0, 10);
}
/**
* Derives a UI resource ID that is unique per result snapshot. The frontend indexes conversation
* resources by ID, so two calls that share a base (resourceUri/text) and args but differ in
* structuredContent, text content, _meta, or error state must not collide and overwrite each other.
*/
function deriveResourceId(
base: string,
result: t.MCPToolCallResponse,
toolArgs: unknown,
serverName?: string,
toolName?: string,
): string {
const meta = (result as { _meta?: unknown } | undefined)?._meta;
const parts = [
serverName ?? '',
toolName ?? '',
base,
result?.structuredContent != null ? JSON.stringify(result.structuredContent) : '',
result?.content != null ? JSON.stringify(result.content) : '',
meta != null ? JSON.stringify(meta) : '',
result?.isError === true ? '1' : '',
toolArgs != null ? JSON.stringify(toolArgs) : '',
];
return generateResourceId(parts.join('\x00'));
}
function fitsPersistedAppLimit(resource: UIResource, maxBytes: number): boolean {
try {
return (
Buffer.byteLength(JSON.stringify({ [Tools.ui_resources]: { data: [resource] } }), 'utf8') <=
maxBytes
);
} catch {
return false;
}
}
function getMCPImageDataMaxBytes(): number {
const raw = process.env.MCP_IMAGE_DATA_MAX_BYTES;
if (!raw) {
return DEFAULT_MCP_IMAGE_DATA_MAX_BYTES;
}
const parsed = Number(raw);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : DEFAULT_MCP_IMAGE_DATA_MAX_BYTES;
}
function getBase64Padding(data: string): number {
if (data.endsWith('==')) {
return 2;
}
if (data.endsWith('=')) {
return 1;
}
return 0;
}
function estimateBase64ImageBytes(data: string): number {
const padding = getBase64Padding(data);
return Math.max(0, Math.floor((data.length * 3) / 4) - padding);
}
function isRemoteImageUrl(data: string): boolean {
return data.startsWith('http://') || data.startsWith('https://');
}
function assertImageDataWithinLimit(item: t.ImageContent): void {
if (isRemoteImageUrl(item.data)) {
return;
}
const maxBytes = getMCPImageDataMaxBytes();
const estimatedBytes = estimateBase64ImageBytes(item.data);
if (estimatedBytes <= maxBytes) {
return;
}
throw new Error(
`MCP image result exceeds maximum size of ${maxBytes} bytes: ${estimatedBytes} bytes`,
);
}
const RECOGNIZED_PROVIDERS = new Set([
'google',
'anthropic',
'openai',
'azureopenai',
'openrouter',
'xai',
'deepseek',
'ollama',
'bedrock',
]);
const imageFormatters: Record<string, undefined | t.ImageFormatter> = {
// google: (item) => ({
// type: 'image',
// inlineData: {
// mimeType: item.mimeType,
// data: item.data,
// },
// }),
// anthropic: (item) => ({
// type: 'image',
// source: {
// type: 'base64',
// media_type: item.mimeType,
// data: item.data,
// },
// }),
default: (item) => ({
type: 'image_url',
image_url: {
url: isRemoteImageUrl(item.data) ? item.data : `data:${item.mimeType};base64,${item.data}`,
},
}),
};
function isImageContent(item: t.ToolContentPart): item is t.ImageContent {
return item.type === 'image';
}
const utf8Decoder = new TextDecoder('utf-8', { fatal: true });
/**
* Reads the body an MCP server embedded in a resource result. A server may deliver the same file
* either as `text` or, under the very same schema, as a base64 `blob`, so both halves are unwrapped
* here — reading only `text` leaves a blob-delivered file as bare URI and MIME type metadata.
*
* Image blobs become artifacts, matching standalone image content. Binary blobs are summarized
* rather than emitted, so they never reach the model as base64. A NUL byte marks a payload binary
* on its own, the way git classifies a file: NUL is valid UTF-8, so decoding alone would pass a
* compiled binary through as text.
*/
function readResourceBody(resource: t.ResourceContents): t.ResourceBody {
if ('text' in resource && typeof resource.text === 'string' && resource.text) {
return { text: resource.text };
}
if (!('blob' in resource) || typeof resource.blob !== 'string' || !resource.blob) {
return {};
}
const mimeType = resource.mimeType;
if (mimeType != null && mimeType.toLowerCase().startsWith('image/')) {
return { image: { type: 'image', data: resource.blob, mimeType } };
}
const bytes = Buffer.from(resource.blob, 'base64');
if (bytes.includes(0)) {
return { binaryBytes: bytes.byteLength };
}
try {
const text = utf8Decoder.decode(bytes);
return text ? { text } : {};
} catch {
return { binaryBytes: bytes.byteLength };
}
}
function hasUsableAppBody(resource: t.ResourceContents): boolean {
if ('text' in resource && typeof resource.text === 'string' && resource.text.length > 0) {
return true;
}
if (!('blob' in resource) || typeof resource.blob !== 'string' || !resource.blob) {
return false;
}
const encoded = resource.blob.replace(/[\t\n\f\r ]/g, '');
if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded)) {
return false;
}
try {
const decoded = utf8Decoder.decode(Buffer.from(encoded, 'base64'));
return decoded.length > 0 && !decoded.includes('\0');
} catch {
return false;
}
}
export function selectResolvedAppResource(
contents: t.ResourceContents[] | undefined,
resourceUri: string,
): t.ResourceContents | undefined {
return contents?.find(
(resource) =>
resource.uri === resourceUri &&
isMcpAppMimeType(resource.mimeType) &&
hasUsableAppBody(resource),
);
}
const LINE_BREAKS = /[\r\n\u2028\u2029]+/g;
/**
* Resource metadata renders as a single labeled line, so a line break inside one lets whoever
* controls it — a hostile server, or merely whoever named a file the server relays — close the
* line early and forge further labels, passing attacker text off as another field. Only these
* one-line fields are flattened; resource bodies keep their line breaks, being the payload.
*/
function flattenMetadata(value: string): string {
return value.replace(LINE_BREAKS, ' ');
}
function describeBinaryResource(bytes: number): string {
return `Resource Content: ${bytes} bytes of binary data (omitted; not UTF-8 text)`;
}
function describeResourceLink(item: t.ResourceLink): string[] {
const lines: string[] = [];
if (item.name) {
lines.push(`Resource Name: ${flattenMetadata(item.name)}`);
}
if (item.title) {
lines.push(`Resource Title: ${flattenMetadata(item.title)}`);
}
if (item.description) {
lines.push(`Resource Description: ${flattenMetadata(item.description)}`);
}
if (item.uri) {
lines.push(`Resource URI: ${flattenMetadata(item.uri)}`);
}
if (item.mimeType) {
lines.push(`Resource MIME Type: ${flattenMetadata(item.mimeType)}`);
}
if (typeof item.size === 'number') {
lines.push(`Resource Size: ${item.size} bytes`);
}
return lines;
}
function parseAsString(result: t.MCPToolCallResponse): string {
const content = result?.content ?? [];
if (!content.length) {
return '(No response)';
}
const text = content
.map((item) => {
if (item.type === 'text') {
return item.text;
}
if (item.type === 'resource_link') {
return describeResourceLink(item).join('\n');
}
if (item.type === 'resource') {
const resourceText = [];
// A ui:// HTML body is a whole document meant for the sandbox, never model context.
if (!isRenderableUiResource(item)) {
const body = readResourceBody(item.resource);
if (body.text) {
resourceText.push(body.text);
} else if (body.image) {
assertImageDataWithinLimit(body.image);
resourceText.push(`data:${body.image.mimeType};base64,${body.image.data}`);
} else if (body.binaryBytes != null) {
resourceText.push(describeBinaryResource(body.binaryBytes));
}
}
if (item.resource.uri) {
resourceText.push(`Resource URI: ${flattenMetadata(item.resource.uri)}`);
}
if (item.resource.mimeType != null && item.resource.mimeType) {
resourceText.push(`Type: ${flattenMetadata(item.resource.mimeType)}`);
}
return resourceText.join('\n');
}
if (isImageContent(item)) {
assertImageDataWithinLimit(item);
}
return JSON.stringify(item, null, 2);
})
.filter(Boolean)
.join('\n\n');
return text;
}
function isUiResource(
item: t.ToolContentPart,
): item is Extract<t.ToolContentPart, { type: 'resource' }> {
const uri = item.type === 'resource' ? item.resource?.uri : undefined;
return typeof uri === 'string' && uri.startsWith('ui://');
}
/**
* MCP Apps renders only `ui://` resources whose mime type is HTML (mime omitted defaults to HTML),
* the single renderable resource type the spec defines.
*
* Deliberately wider than `isMcpAppMimeType`: a plain `text/html` `ui://` resource still renders
* through the legacy HTML renderer. Both tiers parse the media type through the same shared helpers, so a
* differently-cased `Text/HTML;profile=mcp-app` cannot be renderable to one and not the other.
*/
export function isRenderableUiResource(item: t.ToolContentPart): boolean {
if (!isUiResource(item)) {
return false;
}
return isHtmlMediaType(resolveMCPUIResourceMimeType(item.resource.mimeType));
}
/**
* Converts MCPToolCallResponse content into a plain-text string plus optional artifacts
* (images, UI resources). All providers receive string content; images are separated into
* artifacts and merged back by the agents package via formatArtifactPayload / formatAnthropicArtifactContent.
*
* @param provider - Used only to distinguish recognized vs. unrecognized providers.
* All recognized providers currently produce identical string output;
* provider-specific artifact merging is delegated to the agents package.
*/
export function formatToolContent(
result: t.MCPToolCallResponse,
provider: t.Provider,
metadata?: {
serverName?: string;
toolName?: string;
resourceUri?: string;
resolvedAppResource?: t.ResourceContents;
serverBinding?: string;
toolArgs?: Record<string, unknown>;
mcpApps?: TMCPAppsPolicy;
},
): t.FormattedContentResult {
const mcpApps = metadata?.mcpApps ?? resolveMCPAppsPolicy();
const isRecognizedProvider = RECOGNIZED_PROVIDERS.has(provider);
// Truthiness, not != null: an empty resourceUri/serverName/toolName cannot address an app, and a
// single predicate keeps this gate and the synthesis below from drifting apart.
const hasSyntheticApp = !!(
metadata?.resourceUri &&
metadata.serverName &&
metadata.toolName &&
metadata.serverBinding
);
const hasUiResource = result?.content?.some((item) => isUiResource(item));
if (!isRecognizedProvider && !hasSyntheticApp && !hasUiResource) {
return [parseAsString(result), undefined];
}
const content = result?.content ?? [];
if (!content.length && !hasSyntheticApp) {
return ['(No response)', undefined];
}
const imageUrls: t.FormattedContent[] = [];
const uiResources: UIResource[] = [];
let placementMarkerCount = 0;
// Assistants submits only this string; keep its bounded conversion independent from host UI data.
let currentTextBlock = isRecognizedProvider ? '' : parseAsString(result);
type ContentHandler = undefined | ((item: t.ToolContentPart) => void);
const appendTextBlock = (text: string): void => {
currentTextBlock += (currentTextBlock ? '\n\n' : '') + text;
};
const collectImage = (item: t.ImageContent): void => {
assertImageDataWithinLimit(item);
const formatter = imageFormatters.default as t.ImageFormatter;
const formattedImage = formatter(item);
if (formattedImage.type === 'image_url') {
imageUrls.push(formattedImage);
}
};
const contentHandlers: {
text: (item: Extract<t.ToolContentPart, { type: 'text' }>) => void;
image: (item: t.ToolContentPart) => void;
resource: (item: Extract<t.ToolContentPart, { type: 'resource' }>) => void;
resource_link: (item: t.ResourceLink) => void;
} = {
text: (item) => {
if (isRecognizedProvider) {
appendTextBlock(item.text);
}
},
image: (item) => {
if (!isRecognizedProvider || !isImageContent(item)) {
return;
}
collectImage(item);
},
resource: (item) => {
const isDeclaredEcho = hasSyntheticApp && item.resource.uri === metadata?.resourceUri;
const resourceText: string[] = [];
const inlineText =
'text' in item.resource && typeof item.resource.text === 'string' ? item.resource.text : '';
// App-profile documents are resolved separately from the tool result. Plain HTML remains on
// the legacy renderer, except for an echo of the declared App URI which would duplicate it.
const isRenderableResource = isRenderableUiResource(item);
const isLegacyResource =
mcpApps.legacyHtmlEnabled &&
isRenderableResource &&
!isMcpAppMimeType(item.resource.mimeType);
// Custom clients consume non-HTML UI attachments by marker; MIME/policy still own rendering.
const isCoreUiResource =
isUiResource(item) && !isRenderableResource && !isMcpAppMimeType(item.resource.mimeType);
const hasInlineBody =
!!inlineText ||
('blob' in item.resource && typeof item.resource.blob === 'string' && !!item.resource.blob);
const shouldTransportUiResource =
hasInlineBody && (isCoreUiResource || (isLegacyResource && !isDeclaredEcho));
if (shouldTransportUiResource) {
// Distinct URIs must not collide into one resourceId even when their markup is identical:
// the client indexes conversation resources by id, so the second would overwrite the first.
const baseHash = `${item.resource.uri}\x00${inlineText}`;
const resourceId = deriveResourceId(
baseHash,
result,
metadata?.toolArgs,
undefined,
undefined,
);
const uiResource: UIResource = {
...item.resource,
...(isLegacyResource
? { mimeType: resolveMCPUIResourceMimeType(item.resource.mimeType) }
: {}),
resourceId,
};
uiResources.push(uiResource);
placementMarkerCount += 1;
const placementReference =
`UI Resource ID: ${resourceId}\n` + `UI Resource Marker: \\ui{${resourceId}}`;
if (isRecognizedProvider) {
resourceText.push(placementReference);
} else {
appendTextBlock(placementReference);
}
// A non-HTML ui:// image remains an ordinary image result as well as core UI transport.
if (isRecognizedProvider && isCoreUiResource) {
const body = readResourceBody(item.resource);
if (body.image) {
collectImage(body.image);
}
}
} else if (isRecognizedProvider && !isRenderableResource) {
const body = readResourceBody(item.resource);
if (body.text) {
resourceText.push(`Resource Text: ${body.text}`);
} else if (body.image) {
collectImage(body.image);
} else if (body.binaryBytes != null) {
resourceText.push(describeBinaryResource(body.binaryBytes));
}
}
if (isRecognizedProvider && item.resource.uri.length) {
resourceText.push(`Resource URI: ${flattenMetadata(item.resource.uri)}`);
}
if (isRecognizedProvider && item.resource.mimeType != null && item.resource.mimeType) {
resourceText.push(`Resource MIME Type: ${flattenMetadata(item.resource.mimeType)}`);
}
if (resourceText.length) {
appendTextBlock(resourceText.join('\n'));
}
},
resource_link: (item) => {
if (!isRecognizedProvider) {
return;
}
const lines = describeResourceLink(item);
if (lines.length) {
appendTextBlock(lines.join('\n'));
}
},
};
for (const item of content) {
const handler = contentHandlers[item.type as keyof typeof contentHandlers] as ContentHandler;
if (handler) {
handler(item as never);
} else if (isRecognizedProvider) {
const stringified = JSON.stringify(item, null, 2);
appendTextBlock(stringified);
}
}
// The declared App document is acquired through resources/read and passed separately. Persistence
// admission can discard its body while retaining the bound URI descriptor for a later read.
if (
hasSyntheticApp &&
mcpApps.enabled &&
metadata?.resourceUri &&
metadata.serverName &&
metadata.toolName &&
metadata.serverBinding
) {
const resolvedResource =
metadata.resolvedAppResource?.uri === metadata.resourceUri &&
isMcpAppMimeType(metadata.resolvedAppResource.mimeType) &&
hasUsableAppBody(metadata.resolvedAppResource)
? metadata.resolvedAppResource
: undefined;
let resolvedBody = '';
if (resolvedResource && 'text' in resolvedResource) {
resolvedBody = resolvedResource.text;
} else if (resolvedResource && 'blob' in resolvedResource) {
resolvedBody = resolvedResource.blob;
}
const itemUi = (resolvedResource?._meta as { ui?: Record<string, unknown> } | undefined)?.ui as
| { csp?: UIResource['csp']; permissions?: UIResource['permissions'] }
| undefined;
const snapshot = {
uri: metadata.resourceUri,
mimeType: resolvedResource?.mimeType ?? MCP_APP_MIME_TYPE,
serverName: metadata.serverName,
serverBinding: metadata.serverBinding,
toolName: metadata.toolName,
structuredContent: result?.structuredContent,
content: result?.content,
toolArgs: metadata.toolArgs,
isError: result?.isError,
resultMeta: (result as { _meta?: Record<string, unknown> })?._meta,
};
const fullResource: UIResource = {
...resolvedResource,
...snapshot,
resourceId: deriveResourceId(
`${metadata.resourceUri}\x00${resolvedBody}`,
result,
metadata.toolArgs,
metadata.serverName,
metadata.toolName,
),
csp: itemUi?.csp,
permissions: itemUi?.permissions,
};
const maxBytes = mcpApps.maxPersistedAppBytes ?? DEFAULT_MCP_APP_PERSISTED_BYTES;
if (fitsPersistedAppLimit(fullResource, maxBytes)) {
uiResources.push(fullResource);
} else {
const uriResource: UIResource = {
...snapshot,
resourceId: deriveResourceId(
`${metadata.resourceUri}\x00`,
result,
metadata.toolArgs,
metadata.serverName,
metadata.toolName,
),
};
if (fitsPersistedAppLimit(uriResource, maxBytes)) {
uiResources.push(uriResource);
}
}
}
if (placementMarkerCount > 0) {
const uiInstructions = `
UI Resource Markers Available:
- Each resource above includes a stable ID and a marker hint like \`\\ui{abc123}\`
- You should usually introduce what you're showing before placing the marker
- For a single resource: \\ui{resource-id}
- For multiple resources shown separately: \\ui{resource-id-a} \\ui{resource-id-b}
- For multiple resources in a carousel: \\ui{resource-id-a,resource-id-b,resource-id-c}
- The UI will be rendered inline where you place the marker
- Format: \\ui{resource-id} or \\ui{id1,id2,id3} using the IDs provided above`;
currentTextBlock += uiInstructions;
}
let artifacts: t.Artifacts = undefined;
if (imageUrls.length > 0) {
artifacts = { content: imageUrls };
}
if (uiResources.length > 0) {
artifacts = {
...artifacts,
[Tools.ui_resources]: { data: uiResources },
};
}
return [currentTextBlock || (artifacts !== undefined ? '' : '(No response)'), artifacts];
}