Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📟 feat: Record Safe Upstream Model Failures (#15800)

Dustin Healy7a83b8f5d20e
diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js
index d6935069fc..c82c0ac51b 100644
--- a/api/server/controllers/agents/__tests__/openai.spec.js
+++ b/api/server/controllers/agents/__tests__/openai.spec.js
@@ -164,6 +164,7 @@ jest.mock('@librechat/data-schemas', () => ({
}));
jest.mock('@librechat/agents', () => ({
+ ...jest.requireActual('@librechat/agents'),
Callback: { TOOL_ERROR: 'TOOL_ERROR' },
ToolEndHandler: jest.fn(),
formatAgentMessages: jest.fn().mockReturnValue({
@@ -204,6 +205,8 @@ jest.mock('@librechat/api', () => ({
createRun: jest.fn().mockResolvedValue({
processStream: mockProcessStream,
}),
+ createTerminalRunErrorObserver: (...args) =>
+ jest.requireActual('@librechat/api').createTerminalRunErrorObserver(...args),
applyContextToAgent: (...args) => mockApplyContextToAgent(...args),
buildAgentScopedContext: (...args) => mockBuildAgentScopedContext(...args),
buildInlineMemoryContext: (...args) => mockBuildInlineMemoryContext(...args),
@@ -349,7 +352,7 @@ jest.mock('@librechat/api', () => ({
return await execute(execution);
} catch (error) {
executionError = error;
- if (handleExecutionError) return await handleExecutionError(error);
+ if (handleExecutionError) return await handleExecutionError(error, execution?.signal);
throw error;
} finally {
removeCloseListener();
@@ -1037,6 +1040,28 @@ describe('OpenAIChatCompletionController', () => {
});
describe('safe error logging', () => {
+ it('does not classify a client disconnect as an upstream model error', async () => {
+ const api = require('@librechat/api');
+ const { logger } = require('@librechat/data-schemas');
+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });
+ mockProcessStream.mockImplementationOnce(async () => {
+ const modelCallback = api.createRun.mock.calls
+ .at(-1)[0]
+ .modelCallbacks.find(({ name }) => name === 'librechat-upstream-model-error-tracker');
+ modelCallback.handleLLMError(abortError);
+ res.once.mock.calls.find(([event]) => event === 'close')[1]();
+ throw abortError;
+ });
+
+ await OpenAIChatCompletionController(req, res);
+
+ expect(mockExecution.signal.aborted).toBe(true);
+ expect(logger.error).not.toHaveBeenCalledWith(
+ '[OpenAI API] Upstream model error',
+ expect.anything(),
+ );
+ });
+
it('logs bounded metadata and returns a raw-free provider error', async () => {
const api = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
@@ -1050,15 +1075,30 @@ describe('OpenAIChatCompletionController', () => {
},
});
req.config.filters = { messages: { pii: {} } };
- mockProcessStream.mockRejectedValueOnce(providerError);
+ mockProcessStream.mockImplementationOnce(async () => {
+ const modelCallback = api.createRun.mock.calls
+ .at(-1)[0]
+ .modelCallbacks.find(({ name }) => name === 'librechat-upstream-model-error-tracker');
+ modelCallback.handleLLMError(providerError);
+ throw new Error('graph failed', { cause: providerError });
+ });
await OpenAIChatCompletionController(req, res);
- expect(mockGetSafeErrorMetadata).toHaveBeenCalledWith(providerError);
const errorLog = logger.error.mock.calls.find(
- ([message]) => message === '[OpenAI API] Error:',
+ ([message]) => message === '[OpenAI API] Upstream model error',
);
- expect(errorLog).toEqual(['[OpenAI API] Error:', { type: 'Error', status: 502 }]);
+ expect(errorLog).toEqual([
+ '[OpenAI API] Upstream model error',
+ {
+ type: 'Error',
+ status: 502,
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ errorOrigin: 'model_provider',
+ errorType: '502',
+ traceId: 'a64360db27015f7d7eadebf78a806ac1',
+ },
+ ]);
expect(JSON.stringify(errorLog)).not.toContain(rawValue);
expect(api.createErrorResponse).toHaveBeenCalledWith(
'An error occurred while processing the request',
@@ -1071,7 +1111,9 @@ describe('OpenAIChatCompletionController', () => {
it('streams a raw-free provider error after headers are sent', async () => {
const api = require('@librechat/api');
+ const { logger } = require('@librechat/data-schemas');
const rawValue = 'PRIVATE-OPENAI-STREAM-PAYLOAD';
+ const providerError = new Error(`Provider echoed ${rawValue}`);
api.validateRequest.mockReturnValueOnce({
request: {
model: 'agent-123',
@@ -1083,7 +1125,13 @@ describe('OpenAIChatCompletionController', () => {
res.flushHeaders.mockImplementationOnce(() => {
res.headersSent = true;
});
- mockProcessStream.mockRejectedValueOnce(new Error(`Provider echoed ${rawValue}`));
+ mockProcessStream.mockImplementationOnce(async () => {
+ api.createRun.mock.calls
+ .at(-1)[0]
+ .modelCallbacks.find(({ name }) => name === 'librechat-upstream-model-error-tracker')
+ .handleLLMError(providerError);
+ throw providerError;
+ });
await OpenAIChatCompletionController(req, res);
@@ -1094,6 +1142,13 @@ describe('OpenAIChatCompletionController', () => {
);
expect(JSON.stringify(api.createChunk.mock.calls)).not.toContain(rawValue);
expect(JSON.stringify(api.writeSSE.mock.calls)).not.toContain(rawValue);
+ expect(logger.error).toHaveBeenCalledWith(
+ '[OpenAI API] Upstream model error',
+ expect.objectContaining({
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ traceId: 'a64360db27015f7d7eadebf78a806ac1',
+ }),
+ );
});
it('preserves the legacy provider error when protection is inactive', async () => {
diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js
index 635e6099df..e639f015e5 100644
--- a/api/server/controllers/agents/__tests__/responses.unit.spec.js
+++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js
@@ -187,6 +187,7 @@ jest.mock('@librechat/data-schemas', () => ({
}));
jest.mock('@librechat/agents', () => ({
+ ...jest.requireActual('@librechat/agents'),
Callback: { TOOL_ERROR: 'TOOL_ERROR' },
ToolEndHandler: jest.fn(),
formatAgentMessages: jest.fn().mockReturnValue({
@@ -228,6 +229,8 @@ jest.mock('@librechat/api', () => ({
createRun: jest.fn().mockResolvedValue({
processStream: jest.fn().mockResolvedValue(undefined),
}),
+ createTerminalRunErrorObserver: (...args) =>
+ jest.requireActual('@librechat/api').createTerminalRunErrorObserver(...args),
buildInitialToolSessions: jest.fn().mockReturnValue(mockInitialSessions),
applyContextToAgent: (...args) => mockApplyContextToAgent(...args),
buildRunToolSet: jest.fn().mockReturnValue(new Set()),
@@ -417,7 +420,7 @@ jest.mock('@librechat/api', () => ({
return await execute(execution);
} catch (error) {
executionError = error;
- if (handleExecutionError) return await handleExecutionError(error);
+ if (handleExecutionError) return await handleExecutionError(error, execution?.signal);
throw error;
} finally {
removeCloseListener();
@@ -1638,6 +1641,29 @@ describe('createResponse controller', () => {
});
describe('safe error logging', () => {
+ it('does not classify a client disconnect as an upstream model error', async () => {
+ const api = require('@librechat/api');
+ const { logger } = require('@librechat/data-schemas');
+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });
+ api.createRun.mockImplementationOnce(async (options) => ({
+ processStream: jest.fn(async () => {
+ options.modelCallbacks
+ .find(({ name }) => name === 'librechat-upstream-model-error-tracker')
+ .handleLLMError(abortError);
+ res.once.mock.calls.find(([event]) => event === 'close')[1]();
+ throw abortError;
+ }),
+ }));
+
+ await createResponse(req, res);
+
+ expect(mockExecution.signal.aborted).toBe(true);
+ expect(logger.error).not.toHaveBeenCalledWith(
+ '[Responses API] Upstream model error',
+ expect.anything(),
+ );
+ });
+
it('logs bounded metadata and returns a raw-free provider error', async () => {
const api = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
@@ -1655,7 +1681,6 @@ describe('createResponse controller', () => {
await createResponse(req, res);
- expect(mockGetSafeErrorMetadata).toHaveBeenCalledWith(providerError);
const errorLog = logger.error.mock.calls.find(
([message]) => message === '[Responses API] Error:',
);
@@ -1670,6 +1695,50 @@ describe('createResponse controller', () => {
expect(JSON.stringify(api.sendResponsesErrorResponse.mock.calls)).not.toContain(rawValue);
});
+ it.each([false, true])(
+ 'classifies a terminal model callback failure and correlates its trace: stream=%s',
+ async (stream) => {
+ const api = require('@librechat/api');
+ const { logger } = require('@librechat/data-schemas');
+ const rawValue = 'PRIVATE-RESPONSES-UPSTREAM-PAYLOAD';
+ const providerError = Object.assign(new Error(`Provider echoed ${rawValue}`), {
+ code: 'ERR_REMOTE',
+ response: { status: 503, data: { prompt: rawValue } },
+ });
+ req.config.filters = { messages: { pii: {} } };
+ api.validateResponseRequest.mockReturnValueOnce({
+ request: { model: 'agent-123', input: 'Hello', stream },
+ });
+ api.createRun.mockImplementationOnce(async (options) => ({
+ processStream: jest.fn(async () => {
+ options.modelCallbacks
+ .find(({ name }) => name === 'librechat-upstream-model-error-tracker')
+ .handleLLMError(providerError);
+ throw new Error('graph failed', { cause: providerError });
+ }),
+ }));
+
+ await createResponse(req, res);
+
+ const errorLog = logger.error.mock.calls.find(
+ ([message]) => message === '[Responses API] Upstream model error',
+ );
+ expect(errorLog).toEqual([
+ '[Responses API] Upstream model error',
+ {
+ type: 'Error',
+ status: 503,
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ errorOrigin: 'model_provider',
+ errorType: '503',
+ traceId: 'da34f2d846b1b1b770afe89e670770d5',
+ },
+ ]);
+ expect(JSON.stringify(errorLog)).not.toContain(rawValue);
+ expect(JSON.stringify(errorLog)).not.toContain('ERR_REMOTE');
+ },
+ );
+
it('preserves the legacy provider error when protection is inactive', async () => {
const api = require('@librechat/api');
const rawValue = 'LEGACY-RESPONSES-PROVIDER-ERROR';
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index d60dedacc3..6c7c0a9a87 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -169,6 +169,8 @@ const {
createContextMetaPublisher,
selectRunContextMetaToPublish,
resolveToolRoleGrants,
+ createTerminalRunErrorObserver,
+ isAgentRunCancellation,
} = require('@librechat/api');
const {
Run,
@@ -4375,6 +4377,12 @@ class AgentClient extends BaseClient {
let run;
/** @type {Promise<(TAttachment | null)[] | undefined>} */
let memoryPromise;
+ const terminalRunError = createTerminalRunErrorObserver({
+ logger,
+ responseMessageId: this.responseMessageId,
+ source: '[api/server/controllers/agents/client.js #sendCompletion]',
+ genericMessage: '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error',
+ });
const appConfig = this.options.req.config;
const balanceConfig = getBalanceConfig(appConfig);
const transactionsConfig = getTransactionsConfig(appConfig);
@@ -4854,6 +4862,7 @@ class AgentClient extends BaseClient {
modelCallbacks: [
modelBoundCallback,
createAgentMemoryCallback(this.attachmentMemoryContext ?? {}),
+ terminalRunError.modelCallback,
],
// This controller implements the full HITL pause/resume lifecycle (handleRunInterrupt
// persists the pending action; the /resume route rebuilds + continues the run), so it
@@ -5076,7 +5085,7 @@ class AgentClient extends BaseClient {
type: ContentTypes.ERROR,
[ContentTypes.ERROR]: err.message,
});
- } else if (abortController.signal.aborted) {
+ } else if (isAgentRunCancellation(err, abortController.signal)) {
logger.debug(
'[api/server/controllers/agents/client.js #sendCompletion] Operation aborted by user',
{ conversationId: this.conversationId, ...getSafeErrorMetadata(err) },
@@ -5101,10 +5110,7 @@ class AgentClient extends BaseClient {
},
);
} else {
- logger.error(
- '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error type',
- getSafeErrorMetadata(err),
- );
+ terminalRunError.log(err, abortController.signal);
const videoError = resolveGoogleVideoError({
error: err,
provider: this.options.agent?.provider,
@@ -5239,6 +5245,12 @@ class AgentClient extends BaseClient {
let config;
/** @type {ReturnType<createRun>} */
let run;
+ const terminalRunError = createTerminalRunErrorObserver({
+ logger,
+ responseMessageId: this.responseMessageId,
+ source: '[api/server/controllers/agents/client.js #resumeCompletion]',
+ genericMessage: '[api/server/controllers/agents/client.js #resumeCompletion] Unhandled error',
+ });
const appConfig = this.options.req.config;
const balanceConfig = getBalanceConfig(appConfig);
const transactionsConfig = getTransactionsConfig(appConfig);
@@ -5615,7 +5627,11 @@ class AgentClient extends BaseClient {
run = await createRun({
agents,
conversationId: this.conversationId,
- modelCallbacks: [modelBoundCallback, attachmentMemoryCallback],
+ modelCallbacks: [
+ modelBoundCallback,
+ attachmentMemoryCallback,
+ terminalRunError.modelCallback,
+ ],
// State (messages, tool calls) is rehydrated from the checkpoint by
// run.resume; createRun only needs the agents to rebuild the graph.
messages: [],
@@ -5758,7 +5774,7 @@ class AgentClient extends BaseClient {
);
throw err;
}
- if (abortController.signal.aborted) {
+ if (isAgentRunCancellation(err, abortController.signal)) {
logger.debug(
'[api/server/controllers/agents/client.js #resumeCompletion] Aborted by user',
{
@@ -5776,10 +5792,7 @@ class AgentClient extends BaseClient {
{ conversationId: this.conversationId },
);
} else {
- logger.error(
- '[api/server/controllers/agents/client.js #resumeCompletion] Unhandled error',
- getSafeErrorMetadata(err),
- );
+ terminalRunError.log(err, abortController.signal);
this.contentParts.push({
type: ContentTypes.ERROR,
[ContentTypes.ERROR]: getUserFacingRequestError(
diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js
index 3aa60dd326..d7c9d99615 100644
--- a/api/server/controllers/agents/client.test.js
+++ b/api/server/controllers/agents/client.test.js
@@ -2858,6 +2858,151 @@ describe('AgentClient - startup telemetry', () => {
);
});
+ it('classifies a terminal chat-model failure without logging provider content', async () => {
+ jest.clearAllMocks();
+ const { logger } = require('@librechat/data-schemas');
+ const { traceIdForMessage } = require('@librechat/api');
+ const privateValue = 'PRIVATE-UPSTREAM-PROVIDER-CONTENT';
+ const providerError = Object.assign(new Error(`Provider echoed ${privateValue}`), {
+ code: 'InternalServerException',
+ response: {
+ status: 500,
+ headers: { authorization: privateValue },
+ data: { prompt: privateValue },
+ },
+ });
+ const graphError = new Error('model execution failed', { cause: providerError });
+ const abortController = new AbortController();
+ const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ mockCreateRun.mockImplementation(async (options) => {
+ const tracker = options.modelCallbacks.find(
+ (callback) => callback.name === 'librechat-upstream-model-error-tracker',
+ );
+ return {
+ Graph: null,
+ processStream: jest.fn(async () => {
+ tracker.handleLLMError(providerError, 'model-run');
+ abortController.abort();
+ throw graphError;
+ }),
+ getCalibrationRatio: jest.fn(() => 0),
+ };
+ });
+ mockIsHITLEnabled.mockReturnValue(false);
+ const client = new AgentClient({
+ req: {
+ user: { id: 'user-123' },
+ body: {},
+ config: {
+ endpoints: { [EModelEndpoint.agents]: {} },
+ filters: { messages: { pii: {} } },
+ },
+ _resumableStreamId: 'conversation-upstream-error',
+ },
+ res: {},
+ agent: {
+ id: 'agent-123',
+ endpoint: EModelEndpoint.openAI,
+ provider: EModelEndpoint.openAI,
+ model_parameters: { model: 'gpt-4' },
+ hide_sequential_outputs: false,
+ },
+ endpointTokenConfig: {},
+ eventHandlers: {},
+ contentParts: [],
+ collectedUsage: [],
+ artifactPromises: [],
+ });
+ client.conversationId = 'conversation-upstream-error';
+ client.responseMessageId = 'response-upstream-error';
+ client.parentMessageId = 'parent-upstream-error';
+ client.recordCollectedUsage = jest.fn().mockResolvedValue();
+
+ await client.chatCompletion({ payload: [], abortController });
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[api/server/controllers/agents/client.js #sendCompletion] Upstream model error',
+ {
+ type: 'Error',
+ status: 500,
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ errorOrigin: 'model_provider',
+ errorType: '500',
+ traceId: traceIdForMessage('response-upstream-error'),
+ },
+ );
+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(privateValue);
+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('InternalServerException');
+ expect(errorSpy).not.toHaveBeenCalledWith(
+ '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error',
+ expect.anything(),
+ );
+ expect(client.contentParts).toContainEqual({
+ type: ContentTypes.ERROR,
+ [ContentTypes.ERROR]: 'An error occurred while processing the request',
+ });
+ errorSpy.mockRestore();
+ });
+
+ it('keeps a later non-provider run failure on the generic error path', async () => {
+ jest.clearAllMocks();
+ const { logger } = require('@librechat/data-schemas');
+ const recoveredProviderError = new Error('recovered provider failure');
+ const checkpointError = new Error('checkpoint failed');
+ const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ mockCreateRun.mockImplementation(async (options) => {
+ const tracker = options.modelCallbacks.find(
+ (callback) => callback.name === 'librechat-upstream-model-error-tracker',
+ );
+ return {
+ Graph: null,
+ processStream: jest.fn(async () => {
+ tracker.handleLLMError(recoveredProviderError, 'recovered-model-run');
+ throw checkpointError;
+ }),
+ getCalibrationRatio: jest.fn(() => 0),
+ };
+ });
+ mockIsHITLEnabled.mockReturnValue(false);
+ const client = new AgentClient({
+ req: {
+ user: { id: 'user-123' },
+ body: {},
+ config: { endpoints: { [EModelEndpoint.agents]: {} } },
+ _resumableStreamId: 'conversation-non-provider-error',
+ },
+ res: {},
+ agent: {
+ id: 'agent-123',
+ endpoint: EModelEndpoint.openAI,
+ provider: EModelEndpoint.openAI,
+ model_parameters: { model: 'gpt-4' },
+ hide_sequential_outputs: false,
+ },
+ endpointTokenConfig: {},
+ eventHandlers: {},
+ contentParts: [],
+ collectedUsage: [],
+ artifactPromises: [],
+ });
+ client.conversationId = 'conversation-non-provider-error';
+ client.responseMessageId = 'response-non-provider-error';
+ client.parentMessageId = 'parent-non-provider-error';
+ client.recordCollectedUsage = jest.fn().mockResolvedValue();
+
+ await client.chatCompletion({ payload: [] });
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error',
+ { type: 'Error' },
+ );
+ expect(errorSpy).not.toHaveBeenCalledWith(
+ '[api/server/controllers/agents/client.js #sendCompletion] Upstream model error',
+ expect.anything(),
+ );
+ errorSpy.mockRestore();
+ });
+
it('cancels current attachment persistence after combined admission rejects the run', async () => {
jest.clearAllMocks();
let attachmentLimitError;
@@ -9469,6 +9614,63 @@ describe('AgentClient - resumeCompletion content protection', () => {
errorSpy.mockRestore();
});
+ it('classifies a terminal resumed model failure and preserves redaction', async () => {
+ const { logger } = require('@librechat/data-schemas');
+ const { traceIdForMessage } = require('@librechat/api');
+ const privateValue = 'PRIVATE-RESUMED-UPSTREAM-CONTENT';
+ const providerError = Object.assign(new Error(`Provider echoed ${privateValue}`), {
+ code: 'PROVIDER_INTERNAL',
+ status: 503,
+ });
+ const abortController = new AbortController();
+ const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ mockCreateRun.mockImplementation(async (options) => {
+ const tracker = options.modelCallbacks.find(
+ (callback) => callback.name === 'librechat-upstream-model-error-tracker',
+ );
+ return {
+ resume: jest.fn(async () => {
+ tracker.handleLLMError(providerError, 'resumed-model-run');
+ abortController.abort();
+ throw providerError;
+ }),
+ getCalibrationRatio: jest.fn(() => 0),
+ };
+ });
+ const context = makeContext({
+ messages: {
+ pii: {
+ fields: ['text'],
+ starterPatterns: ['email'],
+ },
+ },
+ });
+
+ await AgentClient.prototype.resumeCompletion.call(context, {
+ resumeValue: {},
+ abortController,
+ });
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[api/server/controllers/agents/client.js #resumeCompletion] Upstream model error',
+ {
+ type: 'Error',
+ status: 503,
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ errorOrigin: 'model_provider',
+ errorType: '503',
+ traceId: traceIdForMessage('response-123'),
+ },
+ );
+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(privateValue);
+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('PROVIDER_INTERNAL');
+ expect(context.contentParts).toContainEqual({
+ type: ContentTypes.ERROR,
+ [ContentTypes.ERROR]: 'An error occurred while resuming the request',
+ });
+ errorSpy.mockRestore();
+ });
+
it('preserves provider error detail when content protection is disabled', async () => {
const providerMessage = 'Legacy provider detail';
mockCreateRun.mockRejectedValue(new Error(providerMessage));
diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js
index 1ff29ca874..36713ee168 100644
--- a/api/server/controllers/agents/openai.js
+++ b/api/server/controllers/agents/openai.js
@@ -67,6 +67,7 @@ const {
executeAgentRun,
waitForAgentExecutionWrites,
resolveToolRoleGrants,
+ createTerminalRunErrorObserver,
} = require('@librechat/api');
const {
buildSummarizationHandlers,
@@ -232,7 +233,6 @@ function sendErrorResponse(res, statusCode, message, type = 'invalid_request_err
}
function handleExecutionError({ error, res, context, appConfig }) {
- logger.error('[OpenAI API] Error:', getSafeErrorMetadata(error));
const protectionEnabled = hasModelBoundContentProtection(
appConfig?.filters,
appConfig?.messageFilter?.pii,
@@ -378,6 +378,11 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
}
const responseId = `chatcmpl-${nanoid()}`;
+ const terminalRunError = createTerminalRunErrorObserver({
+ logger,
+ responseMessageId: responseId,
+ source: '[OpenAI API]',
+ });
const created = Math.floor(Date.now() / 1000);
/** @type {import('@librechat/api').OpenAIResponseContext} — key must be `requestId` to match the type used by createChunk/buildNonStreamingResponse */
@@ -430,7 +435,10 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
onSettlementError: (error) => {
logger.error('[OpenAI API] Failed to settle execution:', getSafeErrorMetadata(error));
},
- handleExecutionError: (error) => handleExecutionError({ error, res, context, appConfig }),
+ handleExecutionError: (error, signal) => {
+ terminalRunError.log(error, signal);
+ return handleExecutionError({ error, res, context, appConfig });
+ },
execute: async (execution) => {
if (request.conversation_id != null) {
if (typeof request.conversation_id !== 'string') {
@@ -1128,6 +1136,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
user: { ...createSafeUser(req.user), id: userId },
traceContext: { endpoint: EModelEndpoint.agents },
tenantId: principal.tenantId,
+ modelCallbacks: [terminalRunError.modelCallback],
/** Bills subagent child-run model calls (reported outside the
* streamEvents loop) into the same collectedUsage array. */
subagentUsageSink: createSubagentUsageSink(collectedUsage),
diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js
index bccf4d3dde..367fd04bc0 100644
--- a/api/server/controllers/agents/responses.js
+++ b/api/server/controllers/agents/responses.js
@@ -84,6 +84,7 @@ const {
executeAgentRun,
waitForAgentExecutionWrites,
resolveToolRoleGrants,
+ createTerminalRunErrorObserver,
} = require('@librechat/api');
const {
createResponsesToolEndCallback,
@@ -124,7 +125,6 @@ const filterFilesByRemoteAgentAccess = (params) =>
filterFilesByAgentAccess({ ...params, resourceType: ResourceType.REMOTE_AGENT });
function handleExecutionError({ error, res, appConfig }) {
- logger.error('[Responses API] Error:', getSafeErrorMetadata(error));
const protectionEnabled = hasModelBoundContentProtection(
appConfig?.filters,
appConfig?.messageFilter?.pii,
@@ -622,6 +622,11 @@ const executeResponse = async (envelope, { req, res }) => {
// Generate IDs
const responseId = generateResponseId();
+ const terminalRunError = createTerminalRunErrorObserver({
+ logger,
+ responseMessageId: responseId,
+ source: '[Responses API]',
+ });
const context = createResponseContext(request, responseId);
logger.debug(
@@ -667,7 +672,10 @@ const executeResponse = async (envelope, { req, res }) => {
onSettlementError: (error) => {
logger.error('[Responses API] Failed to settle execution:', getSafeErrorMetadata(error));
},
- handleExecutionError: (error) => handleExecutionError({ error, res, appConfig }),
+ handleExecutionError: (error, signal) => {
+ terminalRunError.log(error, signal);
+ return handleExecutionError({ error, res, appConfig });
+ },
execute: async (execution) => {
if (request.previous_response_id != null) {
if (typeof request.previous_response_id !== 'string') {
@@ -1246,6 +1254,7 @@ const executeResponse = async (envelope, { req, res }) => {
user: { ...createSafeUser(req.user), id: userId },
traceContext: { endpoint: EModelEndpoint.agents },
tenantId: principal.tenantId,
+ modelCallbacks: [terminalRunError.modelCallback],
/** Bills subagent child-run model calls (reported outside the
* streamEvents loop) into the same collectedUsage array. */
subagentUsageSink: createSubagentUsageSink(collectedUsage),
@@ -1481,6 +1490,7 @@ const executeResponse = async (envelope, { req, res }) => {
user: { ...createSafeUser(req.user), id: userId },
traceContext: { endpoint: EModelEndpoint.agents },
tenantId: principal.tenantId,
+ modelCallbacks: [terminalRunError.modelCallback],
/** Bills subagent child-run model calls (reported outside the
* streamEvents loop) into the same collectedUsage array. */
subagentUsageSink: createSubagentUsageSink(collectedUsage),
diff --git a/packages/api/src/agents/failures/index.ts b/packages/api/src/agents/failures/index.ts
new file mode 100644
index 0000000000..973bc5c65c
--- /dev/null
+++ b/packages/api/src/agents/failures/index.ts
@@ -0,0 +1,2 @@
+export * from './terminal';
+export * from './tracker';
diff --git a/packages/api/src/agents/failures/terminal.spec.ts b/packages/api/src/agents/failures/terminal.spec.ts
new file mode 100644
index 0000000000..6e3660cae9
--- /dev/null
+++ b/packages/api/src/agents/failures/terminal.spec.ts
@@ -0,0 +1,102 @@
+import { createTerminalRunErrorObserver, getUpstreamModelErrorMetadata } from './terminal';
+
+describe('terminal agent-run error logging', () => {
+ it('logs stable upstream metadata and deterministic trace correlation', () => {
+ const logger = { error: jest.fn() };
+ const privateValue = 'PRIVATE-PROVIDER-CONTENT';
+ const providerError = Object.assign(new Error(`Provider echoed ${privateValue}`), {
+ name: 'InternalServerException',
+ code: 'InternalServerException',
+ response: {
+ status: 500,
+ headers: { authorization: privateValue },
+ data: { prompt: privateValue },
+ },
+ });
+ const observer = createTerminalRunErrorObserver({
+ logger,
+ responseMessageId: '78847296-b174-4127-a342-78efa427d4a5',
+ source: '[Agent API]',
+ });
+ observer.modelCallback.handleLLMError(providerError);
+
+ observer.log(new Error('graph failed', { cause: providerError }));
+
+ expect(logger.error).toHaveBeenCalledWith('[Agent API] Upstream model error', {
+ type: 'Error',
+ status: 500,
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ errorOrigin: 'model_provider',
+ errorType: '500',
+ traceId: '3a90048362ec9a2e717c6b77769b9a54',
+ });
+ expect(JSON.stringify(logger.error.mock.calls)).not.toContain(privateValue);
+ expect(JSON.stringify(logger.error.mock.calls)).not.toContain('InternalServerException');
+ });
+
+ it('keeps unrelated terminal failures on the generic safe path', () => {
+ const logger = { error: jest.fn() };
+ const observer = createTerminalRunErrorObserver({
+ logger,
+ responseMessageId: 'response-123',
+ source: '[Agent API]',
+ });
+ observer.modelCallback.handleLLMError(new Error('recovered model attempt'));
+
+ observer.log(new Error('checkpoint failed'));
+
+ expect(logger.error).toHaveBeenCalledWith('[Agent API] Error:', { type: 'Error' });
+ });
+
+ it('does not log a tracked client cancellation as an upstream failure', () => {
+ const logger = { error: jest.fn() };
+ const observer = createTerminalRunErrorObserver({ logger, source: '[Agent API]' });
+ const controller = new AbortController();
+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });
+ observer.modelCallback.handleLLMError(abortError);
+ controller.abort();
+
+ observer.log(abortError, controller.signal);
+
+ expect(logger.error).not.toHaveBeenCalled();
+ });
+
+ it('keeps provider AbortErrors observable while the run signal is live', () => {
+ const logger = { error: jest.fn() };
+ const observer = createTerminalRunErrorObserver({ logger, source: '[Agent API]' });
+ const abortError = Object.assign(new Error('provider aborted'), { name: 'AbortError' });
+ observer.modelCallback.handleLLMError(abortError);
+
+ observer.log(abortError, new AbortController().signal);
+
+ expect(logger.error).toHaveBeenCalledWith(
+ '[Agent API] Upstream model error',
+ expect.objectContaining({ errorCode: 'UPSTREAM_MODEL_ERROR' }),
+ );
+ });
+
+ it('keeps real provider failures observable when Stop wins the same-tick race', () => {
+ const logger = { error: jest.fn() };
+ const observer = createTerminalRunErrorObserver({ logger, source: '[Agent API]' });
+ const controller = new AbortController();
+ const providerError = new Error('provider failed');
+ observer.modelCallback.handleLLMError(providerError);
+ controller.abort();
+
+ observer.log(providerError, controller.signal);
+
+ expect(logger.error).toHaveBeenCalledWith(
+ '[Agent API] Upstream model error',
+ expect.objectContaining({ errorCode: 'UPSTREAM_MODEL_ERROR' }),
+ );
+ });
+
+ it('uses a bounded fallback type and omits unavailable trace correlation', () => {
+ expect(getUpstreamModelErrorMetadata(new Error('provider failed'))).toEqual({
+ type: 'Error',
+ errorCode: 'UPSTREAM_MODEL_ERROR',
+ errorOrigin: 'model_provider',
+ errorType: '_OTHER',
+ });
+ });
+});
diff --git a/packages/api/src/agents/failures/terminal.ts b/packages/api/src/agents/failures/terminal.ts
new file mode 100644
index 0000000000..beda93779a
--- /dev/null
+++ b/packages/api/src/agents/failures/terminal.ts
@@ -0,0 +1,81 @@
+import type { SafeErrorMetadata } from '../../utils/errors';
+import type { ModelErrorTrackerCallback } from './tracker';
+import { getSafeErrorMetadata, isAbortError } from '../../utils/errors';
+import { traceIdForMessage } from '../../langfuse/trace';
+import { createModelErrorTracker } from './tracker';
+
+const UPSTREAM_MODEL_ERROR_CODE = 'UPSTREAM_MODEL_ERROR';
+const UPSTREAM_MODEL_ERROR_ORIGIN = 'model_provider';
+const UNKNOWN_UPSTREAM_MODEL_ERROR_TYPE = '_OTHER';
+
+export interface UpstreamModelErrorMetadata extends SafeErrorMetadata {
+ readonly errorCode: typeof UPSTREAM_MODEL_ERROR_CODE;
+ readonly errorOrigin: typeof UPSTREAM_MODEL_ERROR_ORIGIN;
+ readonly errorType: string;
+ readonly traceId?: string;
+}
+
+export interface TerminalRunErrorLogger {
+ error(message: string, metadata: SafeErrorMetadata | UpstreamModelErrorMetadata): void;
+}
+
+export interface TerminalRunErrorObserver {
+ readonly modelCallback: ModelErrorTrackerCallback;
+ readonly log: (error: unknown, signal?: AbortSignal) => void;
+}
+
+/** A run cancellation requires both host-owned abort state and an abort-shaped rejection. */
+export function isAgentRunCancellation(error: unknown, signal?: AbortSignal): boolean {
+ return signal?.aborted === true && isAbortError(error);
+}
+
+export function getUpstreamModelErrorMetadata(
+ error: unknown,
+ responseMessageId?: string,
+): UpstreamModelErrorMetadata {
+ const safeMetadata = getSafeErrorMetadata(error);
+ return {
+ ...safeMetadata,
+ errorCode: UPSTREAM_MODEL_ERROR_CODE,
+ errorOrigin: UPSTREAM_MODEL_ERROR_ORIGIN,
+ errorType:
+ safeMetadata.status != null ? String(safeMetadata.status) : UNKNOWN_UPSTREAM_MODEL_ERROR_TYPE,
+ ...(typeof responseMessageId === 'string' && responseMessageId !== ''
+ ? { traceId: traceIdForMessage(responseMessageId) }
+ : {}),
+ };
+}
+
+/** Applies one terminal-run log taxonomy to every agent ingress. */
+export function createTerminalRunErrorObserver({
+ logger,
+ responseMessageId,
+ source,
+ genericMessage = `${source} Error:`,
+}: {
+ logger: TerminalRunErrorLogger;
+ responseMessageId?: string;
+ source: string;
+ genericMessage?: string;
+}): TerminalRunErrorObserver {
+ const modelErrorTracker = createModelErrorTracker();
+ return Object.freeze({
+ modelCallback: modelErrorTracker.callback,
+ log(error: unknown, signal?: AbortSignal) {
+ if (isAgentRunCancellation(error, signal)) {
+ return;
+ }
+
+ const upstreamModelError = modelErrorTracker.getUpstreamModelError(error);
+ if (upstreamModelError == null) {
+ logger.error(genericMessage, getSafeErrorMetadata(error));
+ return;
+ }
+
+ logger.error(
+ `${source} Upstream model error`,
+ getUpstreamModelErrorMetadata(upstreamModelError, responseMessageId),
+ );
+ },
+ });
+}
diff --git a/packages/api/src/agents/failures/tracker.spec.ts b/packages/api/src/agents/failures/tracker.spec.ts
new file mode 100644
index 0000000000..41b3ea4cf3
--- /dev/null
+++ b/packages/api/src/agents/failures/tracker.spec.ts
@@ -0,0 +1,143 @@
+import { createModelErrorTracker } from './tracker';
+
+describe('createModelErrorTracker', () => {
+ it('returns the exact model error that terminates the graph', () => {
+ const tracker = createModelErrorTracker();
+ const error = new Error('provider failed');
+
+ tracker.callback.handleLLMError(error);
+
+ expect(tracker.getUpstreamModelError(error)).toBe(error);
+ });
+
+ it('returns the tracked error through standard cause and aggregate chains', () => {
+ const tracker = createModelErrorTracker();
+ const providerError = new Error('provider failed');
+ tracker.callback.handleLLMError(providerError);
+
+ const terminal = new Error('graph failed', {
+ cause: { errors: [new Error('other failure'), providerError] },
+ });
+
+ expect(tracker.getUpstreamModelError(terminal)).toBe(providerError);
+ });
+
+ it('does not classify a later graph failure after a recovered model attempt', () => {
+ const tracker = createModelErrorTracker();
+ tracker.callback.handleLLMError(new Error('recovered provider attempt'));
+
+ expect(tracker.getUpstreamModelError(new Error('checkpoint failed'))).toBeNull();
+ });
+
+ it('returns the terminal fallback error when multiple attempts were observed', () => {
+ const tracker = createModelErrorTracker();
+ const primaryError = new Error('primary failed');
+ const fallbackError = new Error('fallback failed');
+ tracker.callback.handleLLMError(primaryError);
+ tracker.callback.handleLLMError(fallbackError);
+
+ expect(tracker.getUpstreamModelError(fallbackError)).toBe(fallbackError);
+ });
+
+ it('fails closed for primitive errors and throwing properties', () => {
+ const tracker = createModelErrorTracker();
+ tracker.callback.handleLLMError('provider failed');
+ const hostile = {};
+ Object.defineProperties(hostile, {
+ cause: {
+ get() {
+ throw new Error('cause getter failed');
+ },
+ },
+ errors: {
+ get() {
+ throw new Error('errors getter failed');
+ },
+ },
+ });
+
+ expect(tracker.getUpstreamModelError('provider failed')).toBeNull();
+ expect(tracker.getUpstreamModelError(hostile)).toBeNull();
+ });
+
+ it('terminates on cyclic error graphs', () => {
+ const tracker = createModelErrorTracker();
+ const first: Error & { cause?: unknown } = new Error('first');
+ const second: Error & { cause?: unknown } = new Error('second');
+ first.cause = second;
+ second.cause = first;
+
+ expect(tracker.getUpstreamModelError(first)).toBeNull();
+ });
+
+ it('caps property reads on an excessively deep cause chain', () => {
+ const tracker = createModelErrorTracker();
+ let propertyReads = 0;
+ let cause: object | undefined;
+ for (let depth = 0; depth < 1_000; depth += 1) {
+ const nextCause = cause;
+ cause = Object.defineProperties(
+ {},
+ {
+ cause: {
+ get() {
+ propertyReads += 1;
+ return nextCause;
+ },
+ },
+ errors: {
+ get() {
+ propertyReads += 1;
+ return undefined;
+ },
+ },
+ },
+ );
+ }
+
+ expect(tracker.getUpstreamModelError(cause)).toBeNull();
+ expect(propertyReads).toBeLessThanOrEqual(64);
+ });
+
+ it('caps aggregate entry reads across a hostile wide graph', () => {
+ const tracker = createModelErrorTracker();
+ let indexReads = 0;
+ const errors = new Proxy(
+ Array.from({ length: 100_000 }, () => null),
+ {
+ get(target, property, receiver) {
+ if (typeof property === 'string' && /^\d+$/.test(property)) {
+ indexReads += 1;
+ }
+ return Reflect.get(target, property, receiver);
+ },
+ },
+ );
+
+ expect(tracker.getUpstreamModelError({ errors })).toBeNull();
+ expect(indexReads).toBeLessThanOrEqual(32);
+ });
+
+ it('prioritizes a deep cause chain over aggregate breadth', () => {
+ const tracker = createModelErrorTracker();
+ const providerError = new Error('provider failed');
+ tracker.callback.handleLLMError(providerError);
+ const errors = Array.from({ length: 1_000 }, () => new Error('aggregate failure'));
+
+ let terminal: Error & { cause?: unknown } = new Error('terminal');
+ const root = terminal;
+ for (let depth = 0; depth < 29; depth += 1) {
+ const next: Error & { cause?: unknown } = new Error(`wrapper ${depth}`);
+ terminal.cause = next;
+ terminal = next;
+ }
+ terminal.cause = providerError;
+
+ expect(
+ tracker.getUpstreamModelError({
+ errors,
+ cause: root,
+ }),
+ ).toBe(providerError);
+ });
+});
diff --git a/packages/api/src/agents/failures/tracker.ts b/packages/api/src/agents/failures/tracker.ts
new file mode 100644
index 0000000000..832112440f
--- /dev/null
+++ b/packages/api/src/agents/failures/tracker.ts
@@ -0,0 +1,116 @@
+const MAX_ERROR_GRAPH_NODES = 32;
+const MAX_AGGREGATE_ENTRIES_INSPECTED = 32;
+
+type ObjectLike = object | ((...args: never[]) => unknown);
+
+export interface ModelErrorTrackerCallback {
+ readonly name: 'librechat-upstream-model-error-tracker';
+ readonly awaitHandlers: true;
+ readonly handleLLMError: (error: unknown) => void;
+}
+
+export interface ModelErrorTracker {
+ readonly callback: ModelErrorTrackerCallback;
+ readonly getUpstreamModelError: (error: unknown) => object | null;
+}
+
+function isObjectLike(value: unknown): value is ObjectLike {
+ return (typeof value === 'object' && value !== null) || typeof value === 'function';
+}
+
+function readProperty(value: ObjectLike, property: PropertyKey): unknown {
+ try {
+ return Reflect.get(value, property);
+ } catch {
+ return undefined;
+ }
+}
+
+function isArray(value: unknown): value is unknown[] {
+ try {
+ return Array.isArray(value);
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * Tracks errors observed at the chat-model callback boundary. Hosts consult
+ * the tracker only after the complete graph rejects, so retries and fallbacks
+ * that recover never classify an otherwise successful run as upstream.
+ */
+export function createModelErrorTracker(): ModelErrorTracker {
+ const modelErrors = new WeakSet<object>();
+
+ const callback: ModelErrorTrackerCallback = Object.freeze({
+ name: 'librechat-upstream-model-error-tracker',
+ awaitHandlers: true,
+ handleLLMError(error: unknown) {
+ if (isObjectLike(error)) {
+ modelErrors.add(error);
+ }
+ },
+ });
+
+ const getUpstreamModelError = (error: unknown): object | null => {
+ if (!isObjectLike(error)) {
+ return null;
+ }
+
+ const pendingCauses: ObjectLike[] = [error];
+ const pendingAggregates: ObjectLike[] = [];
+ const queued = new WeakSet<object>([error]);
+ let visitedCount = 0;
+ let aggregateEntriesInspected = 0;
+
+ while (
+ (pendingCauses.length > 0 || pendingAggregates.length > 0) &&
+ visitedCount < MAX_ERROR_GRAPH_NODES
+ ) {
+ const current = pendingCauses.pop() ?? pendingAggregates.pop();
+ if (current == null) {
+ continue;
+ }
+ visitedCount += 1;
+
+ if (modelErrors.has(current)) {
+ return current;
+ }
+
+ const cause = readProperty(current, 'cause');
+ const errors = readProperty(current, 'errors');
+
+ /** Aggregate entries use a secondary worklist, so even a maximally wide
+ * aggregate cannot consume the node budget before the standard cause
+ * chain is examined. */
+ if (isArray(errors)) {
+ const length = readProperty(errors, 'length');
+ const boundedLength =
+ typeof length === 'number' && Number.isSafeInteger(length) && length > 0
+ ? Math.min(length, MAX_AGGREGATE_ENTRIES_INSPECTED)
+ : 0;
+ for (
+ let index = boundedLength - 1;
+ index >= 0 && aggregateEntriesInspected < MAX_AGGREGATE_ENTRIES_INSPECTED;
+ index -= 1
+ ) {
+ aggregateEntriesInspected += 1;
+ const nested = readProperty(errors, index);
+ if (isObjectLike(nested) && !queued.has(nested)) {
+ queued.add(nested);
+ pendingAggregates.push(nested);
+ }
+ }
+ }
+
+ if (isObjectLike(cause) && !queued.has(cause)) {
+ queued.add(cause);
+ pendingCauses.push(cause);
+ }
+ }
+
+ return null;
+ };
+
+ return Object.freeze({ callback, getUpstreamModelError });
+}
diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts
index 37d3042614..56090134f1 100644
--- a/packages/api/src/agents/index.ts
+++ b/packages/api/src/agents/index.ts
@@ -32,6 +32,7 @@ export * from './lazySubagents';
export * from './lazyHistory';
export * from './memory';
export * from './management';
+export * from './failures';
export * from './reads';
export * from './updates';
export * from './mcpIdentity';
diff --git a/packages/api/src/agents/remote/host.spec.ts b/packages/api/src/agents/remote/host.spec.ts
index b327a0aaef..49746e4b27 100644
--- a/packages/api/src/agents/remote/host.spec.ts
+++ b/packages/api/src/agents/remote/host.spec.ts
@@ -130,6 +130,38 @@ describe('executeAgentRun', () => {
expect(events).toEqual(['rendered', 'settled']);
});
+ it('passes the canonical aborted signal to execution error handling', async () => {
+ let closeListener: (() => void) | undefined;
+ const handleExecutionError = jest.fn<string, [unknown, AbortSignal?]>(() => 'handled');
+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });
+
+ await expect(
+ executeAgentRun({
+ envelope: createEnvelope(),
+ runId: 'run-1',
+ conversationId: 'conversation-1',
+ connection: {
+ isClosed: () => false,
+ onClose: (listener) => {
+ closeListener = listener;
+ return () => undefined;
+ },
+ },
+ isPrincipalActive: async () => true,
+ execute: async () => {
+ closeListener?.();
+ throw abortError;
+ },
+ handleExecutionError,
+ }),
+ ).resolves.toBe('handled');
+
+ expect(handleExecutionError).toHaveBeenCalledWith(abortError, expect.any(AbortSignal));
+ const executionSignal = handleExecutionError.mock.calls[0]?.[1];
+ expect(executionSignal).toBeDefined();
+ expect(executionSignal?.aborted).toBe(true);
+ });
+
it('settles even when trailing-write registration fails', async () => {
const failure = new Error('trailing write registration failed');
diff --git a/packages/api/src/agents/remote/host.ts b/packages/api/src/agents/remote/host.ts
index 1522a2a139..a20997f5d1 100644
--- a/packages/api/src/agents/remote/host.ts
+++ b/packages/api/src/agents/remote/host.ts
@@ -15,7 +15,7 @@ export interface ExecuteAgentRunParams<Result> {
connection?: AgentExecutionConnection;
isPrincipalActive: (userId: string) => Promise<boolean>;
execute: (execution: AgentExecutionEnrollment) => Promise<Result>;
- handleExecutionError?: (error: unknown) => Result | Promise<Result>;
+ handleExecutionError?: (error: unknown, signal?: AbortSignal) => Result | Promise<Result>;
beforeSettle?: (
execution: AgentExecutionEnrollment,
executionError: unknown,
@@ -66,7 +66,7 @@ export async function executeAgentRun<Result>({
} catch (error) {
executionError = error;
if (handleExecutionError != null) {
- return await handleExecutionError(error);
+ return await handleExecutionError(error, execution?.signal);
}
throw error;
} finally {
diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts
index 21fb29c03a..2c95d4d5e6 100644
--- a/packages/api/src/agents/run.ts
+++ b/packages/api/src/agents/run.ts
@@ -50,6 +50,7 @@ import type { CallbackHandlerMethods } from '@langchain/core/callbacks/base';
import type { BaseMessage } from '@librechat/agents/langchain/messages';
import type { Callbacks } from '@langchain/core/callbacks/manager';
import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent';
+import type { ModelErrorTrackerCallback } from '~/agents/failures/tracker';
import type { ToolInputValidationError } from '~/agents/toolValidation';
import type { ResolvedToolApprovalHook } from '~/agents/hitl/hooks';
import type { TerminalSteerHook } from '~/agents/steering/runtime';
@@ -986,6 +987,8 @@ type CallbackClientOptions = {
fallbacks?: FallbackConfig[];
};
+type RunModelCallback = ModelBoundChatModelCallback | ModelErrorTrackerCallback;
+
/**
* Installs run-stable callbacks on the model client itself. Subagent child
* graphs intentionally replace invocation callbacks with their own event
@@ -994,7 +997,7 @@ type CallbackClientOptions = {
*/
function withModelCallbacks<T extends object>(
options: T,
- modelCallbacks: readonly ModelBoundChatModelCallback[] | undefined,
+ modelCallbacks: readonly RunModelCallback[] | undefined,
): T {
if (!modelCallbacks?.length) {
return options;
@@ -1640,8 +1643,8 @@ export async function createRun({
compactionSemanticIndex?: CompactionSemanticIndex;
/** Cross-run summary from formatAgentMessages, forwarded to AgentContext */
initialSummary?: { text: string; tokenCount: number };
- /** Model-level guards inherited by root, summary, fallback, and subagent clients. */
- modelCallbacks?: readonly ModelBoundChatModelCallback[];
+ /** Model callbacks inherited by root, summary, fallback, and subagent clients. */
+ modelCallbacks?: readonly RunModelCallback[];
/** Calibration ratio from previous run's contextMeta, seeds the pruner EMA */
calibrationRatio?: number;
/**
diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.ts
index 487369aa86..7f2e4b3573 100644
--- a/packages/api/src/agents/testHook.ts
+++ b/packages/api/src/agents/testHook.ts
@@ -1,7 +1,6 @@
import { logger } from '@librechat/data-schemas';
import type { BaseMessage } from '@librechat/agents/langchain/messages';
import type { Run, IState } from '@librechat/agents';
-import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent';
/**
* Context handed to a test run hook so it can shape fake-model behavior from
@@ -17,7 +16,8 @@ export interface TestRunHookContext {
*/
conversationId?: string;
agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>;
- modelCallbacks?: readonly ModelBoundChatModelCallback[];
+ /** Opaque model-client handlers forwarded to the fake model by the e2e harness. */
+ modelCallbacks?: readonly object[];
}
export type TestRunHook = (run: Run<IState>, context: TestRunHookContext) => void;