Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
📟 feat: Record Safe Upstream Model Failures (#15800)
Dustin Healy7a83b8f5d20e
diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.jsindex d6935069fc..c82c0ac51b 100644--- a/api/server/controllers/agents/__tests__/openai.spec.js+++ b/api/server/controllers/agents/__tests__/openai.spec.js@@ -164,6 +164,7 @@ jest.mock('@librechat/data-schemas', () => ({}));jest.mock('@librechat/agents', () => ({+ ...jest.requireActual('@librechat/agents'),Callback: { TOOL_ERROR: 'TOOL_ERROR' },ToolEndHandler: jest.fn(),formatAgentMessages: jest.fn().mockReturnValue({@@ -204,6 +205,8 @@ jest.mock('@librechat/api', () => ({createRun: jest.fn().mockResolvedValue({processStream: mockProcessStream,}),+ createTerminalRunErrorObserver: (...args) =>+ jest.requireActual('@librechat/api').createTerminalRunErrorObserver(...args),applyContextToAgent: (...args) => mockApplyContextToAgent(...args),buildAgentScopedContext: (...args) => mockBuildAgentScopedContext(...args),buildInlineMemoryContext: (...args) => mockBuildInlineMemoryContext(...args),@@ -349,7 +352,7 @@ jest.mock('@librechat/api', () => ({return await execute(execution);} catch (error) {executionError = error;- if (handleExecutionError) return await handleExecutionError(error);+ if (handleExecutionError) return await handleExecutionError(error, execution?.signal);throw error;} finally {removeCloseListener();@@ -1037,6 +1040,28 @@ describe('OpenAIChatCompletionController', () => {});describe('safe error logging', () => {+ it('does not classify a client disconnect as an upstream model error', async () => {+ const api = require('@librechat/api');+ const { logger } = require('@librechat/data-schemas');+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });+ mockProcessStream.mockImplementationOnce(async () => {+ const modelCallback = api.createRun.mock.calls+ .at(-1)[0]+ .modelCallbacks.find(({ name }) => name === 'librechat-upstream-model-error-tracker');+ modelCallback.handleLLMError(abortError);+ res.once.mock.calls.find(([event]) => event === 'close')[1]();+ throw abortError;+ });++ await OpenAIChatCompletionController(req, res);++ expect(mockExecution.signal.aborted).toBe(true);+ expect(logger.error).not.toHaveBeenCalledWith(+ '[OpenAI API] Upstream model error',+ expect.anything(),+ );+ });+it('logs bounded metadata and returns a raw-free provider error', async () => {const api = require('@librechat/api');const { logger } = require('@librechat/data-schemas');@@ -1050,15 +1075,30 @@ describe('OpenAIChatCompletionController', () => {},});req.config.filters = { messages: { pii: {} } };- mockProcessStream.mockRejectedValueOnce(providerError);+ mockProcessStream.mockImplementationOnce(async () => {+ const modelCallback = api.createRun.mock.calls+ .at(-1)[0]+ .modelCallbacks.find(({ name }) => name === 'librechat-upstream-model-error-tracker');+ modelCallback.handleLLMError(providerError);+ throw new Error('graph failed', { cause: providerError });+ });await OpenAIChatCompletionController(req, res);- expect(mockGetSafeErrorMetadata).toHaveBeenCalledWith(providerError);const errorLog = logger.error.mock.calls.find(- ([message]) => message === '[OpenAI API] Error:',+ ([message]) => message === '[OpenAI API] Upstream model error',);- expect(errorLog).toEqual(['[OpenAI API] Error:', { type: 'Error', status: 502 }]);+ expect(errorLog).toEqual([+ '[OpenAI API] Upstream model error',+ {+ type: 'Error',+ status: 502,+ errorCode: 'UPSTREAM_MODEL_ERROR',+ errorOrigin: 'model_provider',+ errorType: '502',+ traceId: 'a64360db27015f7d7eadebf78a806ac1',+ },+ ]);expect(JSON.stringify(errorLog)).not.toContain(rawValue);expect(api.createErrorResponse).toHaveBeenCalledWith('An error occurred while processing the request',@@ -1071,7 +1111,9 @@ describe('OpenAIChatCompletionController', () => {it('streams a raw-free provider error after headers are sent', async () => {const api = require('@librechat/api');+ const { logger } = require('@librechat/data-schemas');const rawValue = 'PRIVATE-OPENAI-STREAM-PAYLOAD';+ const providerError = new Error(`Provider echoed ${rawValue}`);api.validateRequest.mockReturnValueOnce({request: {model: 'agent-123',@@ -1083,7 +1125,13 @@ describe('OpenAIChatCompletionController', () => {res.flushHeaders.mockImplementationOnce(() => {res.headersSent = true;});- mockProcessStream.mockRejectedValueOnce(new Error(`Provider echoed ${rawValue}`));+ mockProcessStream.mockImplementationOnce(async () => {+ api.createRun.mock.calls+ .at(-1)[0]+ .modelCallbacks.find(({ name }) => name === 'librechat-upstream-model-error-tracker')+ .handleLLMError(providerError);+ throw providerError;+ });await OpenAIChatCompletionController(req, res);@@ -1094,6 +1142,13 @@ describe('OpenAIChatCompletionController', () => {);expect(JSON.stringify(api.createChunk.mock.calls)).not.toContain(rawValue);expect(JSON.stringify(api.writeSSE.mock.calls)).not.toContain(rawValue);+ expect(logger.error).toHaveBeenCalledWith(+ '[OpenAI API] Upstream model error',+ expect.objectContaining({+ errorCode: 'UPSTREAM_MODEL_ERROR',+ traceId: 'a64360db27015f7d7eadebf78a806ac1',+ }),+ );});it('preserves the legacy provider error when protection is inactive', async () => {diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.jsindex 635e6099df..e639f015e5 100644--- a/api/server/controllers/agents/__tests__/responses.unit.spec.js+++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js@@ -187,6 +187,7 @@ jest.mock('@librechat/data-schemas', () => ({}));jest.mock('@librechat/agents', () => ({+ ...jest.requireActual('@librechat/agents'),Callback: { TOOL_ERROR: 'TOOL_ERROR' },ToolEndHandler: jest.fn(),formatAgentMessages: jest.fn().mockReturnValue({@@ -228,6 +229,8 @@ jest.mock('@librechat/api', () => ({createRun: jest.fn().mockResolvedValue({processStream: jest.fn().mockResolvedValue(undefined),}),+ createTerminalRunErrorObserver: (...args) =>+ jest.requireActual('@librechat/api').createTerminalRunErrorObserver(...args),buildInitialToolSessions: jest.fn().mockReturnValue(mockInitialSessions),applyContextToAgent: (...args) => mockApplyContextToAgent(...args),buildRunToolSet: jest.fn().mockReturnValue(new Set()),@@ -417,7 +420,7 @@ jest.mock('@librechat/api', () => ({return await execute(execution);} catch (error) {executionError = error;- if (handleExecutionError) return await handleExecutionError(error);+ if (handleExecutionError) return await handleExecutionError(error, execution?.signal);throw error;} finally {removeCloseListener();@@ -1638,6 +1641,29 @@ describe('createResponse controller', () => {});describe('safe error logging', () => {+ it('does not classify a client disconnect as an upstream model error', async () => {+ const api = require('@librechat/api');+ const { logger } = require('@librechat/data-schemas');+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });+ api.createRun.mockImplementationOnce(async (options) => ({+ processStream: jest.fn(async () => {+ options.modelCallbacks+ .find(({ name }) => name === 'librechat-upstream-model-error-tracker')+ .handleLLMError(abortError);+ res.once.mock.calls.find(([event]) => event === 'close')[1]();+ throw abortError;+ }),+ }));++ await createResponse(req, res);++ expect(mockExecution.signal.aborted).toBe(true);+ expect(logger.error).not.toHaveBeenCalledWith(+ '[Responses API] Upstream model error',+ expect.anything(),+ );+ });+it('logs bounded metadata and returns a raw-free provider error', async () => {const api = require('@librechat/api');const { logger } = require('@librechat/data-schemas');@@ -1655,7 +1681,6 @@ describe('createResponse controller', () => {await createResponse(req, res);- expect(mockGetSafeErrorMetadata).toHaveBeenCalledWith(providerError);const errorLog = logger.error.mock.calls.find(([message]) => message === '[Responses API] Error:',);@@ -1670,6 +1695,50 @@ describe('createResponse controller', () => {expect(JSON.stringify(api.sendResponsesErrorResponse.mock.calls)).not.toContain(rawValue);});+ it.each([false, true])(+ 'classifies a terminal model callback failure and correlates its trace: stream=%s',+ async (stream) => {+ const api = require('@librechat/api');+ const { logger } = require('@librechat/data-schemas');+ const rawValue = 'PRIVATE-RESPONSES-UPSTREAM-PAYLOAD';+ const providerError = Object.assign(new Error(`Provider echoed ${rawValue}`), {+ code: 'ERR_REMOTE',+ response: { status: 503, data: { prompt: rawValue } },+ });+ req.config.filters = { messages: { pii: {} } };+ api.validateResponseRequest.mockReturnValueOnce({+ request: { model: 'agent-123', input: 'Hello', stream },+ });+ api.createRun.mockImplementationOnce(async (options) => ({+ processStream: jest.fn(async () => {+ options.modelCallbacks+ .find(({ name }) => name === 'librechat-upstream-model-error-tracker')+ .handleLLMError(providerError);+ throw new Error('graph failed', { cause: providerError });+ }),+ }));++ await createResponse(req, res);++ const errorLog = logger.error.mock.calls.find(+ ([message]) => message === '[Responses API] Upstream model error',+ );+ expect(errorLog).toEqual([+ '[Responses API] Upstream model error',+ {+ type: 'Error',+ status: 503,+ errorCode: 'UPSTREAM_MODEL_ERROR',+ errorOrigin: 'model_provider',+ errorType: '503',+ traceId: 'da34f2d846b1b1b770afe89e670770d5',+ },+ ]);+ expect(JSON.stringify(errorLog)).not.toContain(rawValue);+ expect(JSON.stringify(errorLog)).not.toContain('ERR_REMOTE');+ },+ );+it('preserves the legacy provider error when protection is inactive', async () => {const api = require('@librechat/api');const rawValue = 'LEGACY-RESPONSES-PROVIDER-ERROR';diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex d60dedacc3..6c7c0a9a87 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -169,6 +169,8 @@ const {createContextMetaPublisher,selectRunContextMetaToPublish,resolveToolRoleGrants,+ createTerminalRunErrorObserver,+ isAgentRunCancellation,} = require('@librechat/api');const {Run,@@ -4375,6 +4377,12 @@ class AgentClient extends BaseClient {let run;/** @type {Promise<(TAttachment | null)[] | undefined>} */let memoryPromise;+ const terminalRunError = createTerminalRunErrorObserver({+ logger,+ responseMessageId: this.responseMessageId,+ source: '[api/server/controllers/agents/client.js #sendCompletion]',+ genericMessage: '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error',+ });const appConfig = this.options.req.config;const balanceConfig = getBalanceConfig(appConfig);const transactionsConfig = getTransactionsConfig(appConfig);@@ -4854,6 +4862,7 @@ class AgentClient extends BaseClient {modelCallbacks: [modelBoundCallback,createAgentMemoryCallback(this.attachmentMemoryContext ?? {}),+ terminalRunError.modelCallback,],// This controller implements the full HITL pause/resume lifecycle (handleRunInterrupt// persists the pending action; the /resume route rebuilds + continues the run), so it@@ -5076,7 +5085,7 @@ class AgentClient extends BaseClient {type: ContentTypes.ERROR,[ContentTypes.ERROR]: err.message,});- } else if (abortController.signal.aborted) {+ } else if (isAgentRunCancellation(err, abortController.signal)) {logger.debug('[api/server/controllers/agents/client.js #sendCompletion] Operation aborted by user',{ conversationId: this.conversationId, ...getSafeErrorMetadata(err) },@@ -5101,10 +5110,7 @@ class AgentClient extends BaseClient {},);} else {- logger.error(- '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error type',- getSafeErrorMetadata(err),- );+ terminalRunError.log(err, abortController.signal);const videoError = resolveGoogleVideoError({error: err,provider: this.options.agent?.provider,@@ -5239,6 +5245,12 @@ class AgentClient extends BaseClient {let config;/** @type {ReturnType<createRun>} */let run;+ const terminalRunError = createTerminalRunErrorObserver({+ logger,+ responseMessageId: this.responseMessageId,+ source: '[api/server/controllers/agents/client.js #resumeCompletion]',+ genericMessage: '[api/server/controllers/agents/client.js #resumeCompletion] Unhandled error',+ });const appConfig = this.options.req.config;const balanceConfig = getBalanceConfig(appConfig);const transactionsConfig = getTransactionsConfig(appConfig);@@ -5615,7 +5627,11 @@ class AgentClient extends BaseClient {run = await createRun({agents,conversationId: this.conversationId,- modelCallbacks: [modelBoundCallback, attachmentMemoryCallback],+ modelCallbacks: [+ modelBoundCallback,+ attachmentMemoryCallback,+ terminalRunError.modelCallback,+ ],// State (messages, tool calls) is rehydrated from the checkpoint by// run.resume; createRun only needs the agents to rebuild the graph.messages: [],@@ -5758,7 +5774,7 @@ class AgentClient extends BaseClient {);throw err;}- if (abortController.signal.aborted) {+ if (isAgentRunCancellation(err, abortController.signal)) {logger.debug('[api/server/controllers/agents/client.js #resumeCompletion] Aborted by user',{@@ -5776,10 +5792,7 @@ class AgentClient extends BaseClient {{ conversationId: this.conversationId },);} else {- logger.error(- '[api/server/controllers/agents/client.js #resumeCompletion] Unhandled error',- getSafeErrorMetadata(err),- );+ terminalRunError.log(err, abortController.signal);this.contentParts.push({type: ContentTypes.ERROR,[ContentTypes.ERROR]: getUserFacingRequestError(diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.jsindex 3aa60dd326..d7c9d99615 100644--- a/api/server/controllers/agents/client.test.js+++ b/api/server/controllers/agents/client.test.js@@ -2858,6 +2858,151 @@ describe('AgentClient - startup telemetry', () => {);});+ it('classifies a terminal chat-model failure without logging provider content', async () => {+ jest.clearAllMocks();+ const { logger } = require('@librechat/data-schemas');+ const { traceIdForMessage } = require('@librechat/api');+ const privateValue = 'PRIVATE-UPSTREAM-PROVIDER-CONTENT';+ const providerError = Object.assign(new Error(`Provider echoed ${privateValue}`), {+ code: 'InternalServerException',+ response: {+ status: 500,+ headers: { authorization: privateValue },+ data: { prompt: privateValue },+ },+ });+ const graphError = new Error('model execution failed', { cause: providerError });+ const abortController = new AbortController();+ const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);+ mockCreateRun.mockImplementation(async (options) => {+ const tracker = options.modelCallbacks.find(+ (callback) => callback.name === 'librechat-upstream-model-error-tracker',+ );+ return {+ Graph: null,+ processStream: jest.fn(async () => {+ tracker.handleLLMError(providerError, 'model-run');+ abortController.abort();+ throw graphError;+ }),+ getCalibrationRatio: jest.fn(() => 0),+ };+ });+ mockIsHITLEnabled.mockReturnValue(false);+ const client = new AgentClient({+ req: {+ user: { id: 'user-123' },+ body: {},+ config: {+ endpoints: { [EModelEndpoint.agents]: {} },+ filters: { messages: { pii: {} } },+ },+ _resumableStreamId: 'conversation-upstream-error',+ },+ res: {},+ agent: {+ id: 'agent-123',+ endpoint: EModelEndpoint.openAI,+ provider: EModelEndpoint.openAI,+ model_parameters: { model: 'gpt-4' },+ hide_sequential_outputs: false,+ },+ endpointTokenConfig: {},+ eventHandlers: {},+ contentParts: [],+ collectedUsage: [],+ artifactPromises: [],+ });+ client.conversationId = 'conversation-upstream-error';+ client.responseMessageId = 'response-upstream-error';+ client.parentMessageId = 'parent-upstream-error';+ client.recordCollectedUsage = jest.fn().mockResolvedValue();++ await client.chatCompletion({ payload: [], abortController });++ expect(errorSpy).toHaveBeenCalledWith(+ '[api/server/controllers/agents/client.js #sendCompletion] Upstream model error',+ {+ type: 'Error',+ status: 500,+ errorCode: 'UPSTREAM_MODEL_ERROR',+ errorOrigin: 'model_provider',+ errorType: '500',+ traceId: traceIdForMessage('response-upstream-error'),+ },+ );+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(privateValue);+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('InternalServerException');+ expect(errorSpy).not.toHaveBeenCalledWith(+ '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error',+ expect.anything(),+ );+ expect(client.contentParts).toContainEqual({+ type: ContentTypes.ERROR,+ [ContentTypes.ERROR]: 'An error occurred while processing the request',+ });+ errorSpy.mockRestore();+ });++ it('keeps a later non-provider run failure on the generic error path', async () => {+ jest.clearAllMocks();+ const { logger } = require('@librechat/data-schemas');+ const recoveredProviderError = new Error('recovered provider failure');+ const checkpointError = new Error('checkpoint failed');+ const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);+ mockCreateRun.mockImplementation(async (options) => {+ const tracker = options.modelCallbacks.find(+ (callback) => callback.name === 'librechat-upstream-model-error-tracker',+ );+ return {+ Graph: null,+ processStream: jest.fn(async () => {+ tracker.handleLLMError(recoveredProviderError, 'recovered-model-run');+ throw checkpointError;+ }),+ getCalibrationRatio: jest.fn(() => 0),+ };+ });+ mockIsHITLEnabled.mockReturnValue(false);+ const client = new AgentClient({+ req: {+ user: { id: 'user-123' },+ body: {},+ config: { endpoints: { [EModelEndpoint.agents]: {} } },+ _resumableStreamId: 'conversation-non-provider-error',+ },+ res: {},+ agent: {+ id: 'agent-123',+ endpoint: EModelEndpoint.openAI,+ provider: EModelEndpoint.openAI,+ model_parameters: { model: 'gpt-4' },+ hide_sequential_outputs: false,+ },+ endpointTokenConfig: {},+ eventHandlers: {},+ contentParts: [],+ collectedUsage: [],+ artifactPromises: [],+ });+ client.conversationId = 'conversation-non-provider-error';+ client.responseMessageId = 'response-non-provider-error';+ client.parentMessageId = 'parent-non-provider-error';+ client.recordCollectedUsage = jest.fn().mockResolvedValue();++ await client.chatCompletion({ payload: [] });++ expect(errorSpy).toHaveBeenCalledWith(+ '[api/server/controllers/agents/client.js #sendCompletion] Unhandled error',+ { type: 'Error' },+ );+ expect(errorSpy).not.toHaveBeenCalledWith(+ '[api/server/controllers/agents/client.js #sendCompletion] Upstream model error',+ expect.anything(),+ );+ errorSpy.mockRestore();+ });+it('cancels current attachment persistence after combined admission rejects the run', async () => {jest.clearAllMocks();let attachmentLimitError;@@ -9469,6 +9614,63 @@ describe('AgentClient - resumeCompletion content protection', () => {errorSpy.mockRestore();});+ it('classifies a terminal resumed model failure and preserves redaction', async () => {+ const { logger } = require('@librechat/data-schemas');+ const { traceIdForMessage } = require('@librechat/api');+ const privateValue = 'PRIVATE-RESUMED-UPSTREAM-CONTENT';+ const providerError = Object.assign(new Error(`Provider echoed ${privateValue}`), {+ code: 'PROVIDER_INTERNAL',+ status: 503,+ });+ const abortController = new AbortController();+ const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => logger);+ mockCreateRun.mockImplementation(async (options) => {+ const tracker = options.modelCallbacks.find(+ (callback) => callback.name === 'librechat-upstream-model-error-tracker',+ );+ return {+ resume: jest.fn(async () => {+ tracker.handleLLMError(providerError, 'resumed-model-run');+ abortController.abort();+ throw providerError;+ }),+ getCalibrationRatio: jest.fn(() => 0),+ };+ });+ const context = makeContext({+ messages: {+ pii: {+ fields: ['text'],+ starterPatterns: ['email'],+ },+ },+ });++ await AgentClient.prototype.resumeCompletion.call(context, {+ resumeValue: {},+ abortController,+ });++ expect(errorSpy).toHaveBeenCalledWith(+ '[api/server/controllers/agents/client.js #resumeCompletion] Upstream model error',+ {+ type: 'Error',+ status: 503,+ errorCode: 'UPSTREAM_MODEL_ERROR',+ errorOrigin: 'model_provider',+ errorType: '503',+ traceId: traceIdForMessage('response-123'),+ },+ );+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain(privateValue);+ expect(JSON.stringify(errorSpy.mock.calls)).not.toContain('PROVIDER_INTERNAL');+ expect(context.contentParts).toContainEqual({+ type: ContentTypes.ERROR,+ [ContentTypes.ERROR]: 'An error occurred while resuming the request',+ });+ errorSpy.mockRestore();+ });+it('preserves provider error detail when content protection is disabled', async () => {const providerMessage = 'Legacy provider detail';mockCreateRun.mockRejectedValue(new Error(providerMessage));diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.jsindex 1ff29ca874..36713ee168 100644--- a/api/server/controllers/agents/openai.js+++ b/api/server/controllers/agents/openai.js@@ -67,6 +67,7 @@ const {executeAgentRun,waitForAgentExecutionWrites,resolveToolRoleGrants,+ createTerminalRunErrorObserver,} = require('@librechat/api');const {buildSummarizationHandlers,@@ -232,7 +233,6 @@ function sendErrorResponse(res, statusCode, message, type = 'invalid_request_err}function handleExecutionError({ error, res, context, appConfig }) {- logger.error('[OpenAI API] Error:', getSafeErrorMetadata(error));const protectionEnabled = hasModelBoundContentProtection(appConfig?.filters,appConfig?.messageFilter?.pii,@@ -378,6 +378,11 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {}const responseId = `chatcmpl-${nanoid()}`;+ const terminalRunError = createTerminalRunErrorObserver({+ logger,+ responseMessageId: responseId,+ source: '[OpenAI API]',+ });const created = Math.floor(Date.now() / 1000);/** @type {import('@librechat/api').OpenAIResponseContext} — key must be `requestId` to match the type used by createChunk/buildNonStreamingResponse */@@ -430,7 +435,10 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {onSettlementError: (error) => {logger.error('[OpenAI API] Failed to settle execution:', getSafeErrorMetadata(error));},- handleExecutionError: (error) => handleExecutionError({ error, res, context, appConfig }),+ handleExecutionError: (error, signal) => {+ terminalRunError.log(error, signal);+ return handleExecutionError({ error, res, context, appConfig });+ },execute: async (execution) => {if (request.conversation_id != null) {if (typeof request.conversation_id !== 'string') {@@ -1128,6 +1136,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {user: { ...createSafeUser(req.user), id: userId },traceContext: { endpoint: EModelEndpoint.agents },tenantId: principal.tenantId,+ modelCallbacks: [terminalRunError.modelCallback],/** Bills subagent child-run model calls (reported outside the* streamEvents loop) into the same collectedUsage array. */subagentUsageSink: createSubagentUsageSink(collectedUsage),diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.jsindex bccf4d3dde..367fd04bc0 100644--- a/api/server/controllers/agents/responses.js+++ b/api/server/controllers/agents/responses.js@@ -84,6 +84,7 @@ const {executeAgentRun,waitForAgentExecutionWrites,resolveToolRoleGrants,+ createTerminalRunErrorObserver,} = require('@librechat/api');const {createResponsesToolEndCallback,@@ -124,7 +125,6 @@ const filterFilesByRemoteAgentAccess = (params) =>filterFilesByAgentAccess({ ...params, resourceType: ResourceType.REMOTE_AGENT });function handleExecutionError({ error, res, appConfig }) {- logger.error('[Responses API] Error:', getSafeErrorMetadata(error));const protectionEnabled = hasModelBoundContentProtection(appConfig?.filters,appConfig?.messageFilter?.pii,@@ -622,6 +622,11 @@ const executeResponse = async (envelope, { req, res }) => {// Generate IDsconst responseId = generateResponseId();+ const terminalRunError = createTerminalRunErrorObserver({+ logger,+ responseMessageId: responseId,+ source: '[Responses API]',+ });const context = createResponseContext(request, responseId);logger.debug(@@ -667,7 +672,10 @@ const executeResponse = async (envelope, { req, res }) => {onSettlementError: (error) => {logger.error('[Responses API] Failed to settle execution:', getSafeErrorMetadata(error));},- handleExecutionError: (error) => handleExecutionError({ error, res, appConfig }),+ handleExecutionError: (error, signal) => {+ terminalRunError.log(error, signal);+ return handleExecutionError({ error, res, appConfig });+ },execute: async (execution) => {if (request.previous_response_id != null) {if (typeof request.previous_response_id !== 'string') {@@ -1246,6 +1254,7 @@ const executeResponse = async (envelope, { req, res }) => {user: { ...createSafeUser(req.user), id: userId },traceContext: { endpoint: EModelEndpoint.agents },tenantId: principal.tenantId,+ modelCallbacks: [terminalRunError.modelCallback],/** Bills subagent child-run model calls (reported outside the* streamEvents loop) into the same collectedUsage array. */subagentUsageSink: createSubagentUsageSink(collectedUsage),@@ -1481,6 +1490,7 @@ const executeResponse = async (envelope, { req, res }) => {user: { ...createSafeUser(req.user), id: userId },traceContext: { endpoint: EModelEndpoint.agents },tenantId: principal.tenantId,+ modelCallbacks: [terminalRunError.modelCallback],/** Bills subagent child-run model calls (reported outside the* streamEvents loop) into the same collectedUsage array. */subagentUsageSink: createSubagentUsageSink(collectedUsage),diff --git a/packages/api/src/agents/failures/index.ts b/packages/api/src/agents/failures/index.tsnew file mode 100644index 0000000000..973bc5c65c--- /dev/null+++ b/packages/api/src/agents/failures/index.ts@@ -0,0 +1,2 @@+export * from './terminal';+export * from './tracker';diff --git a/packages/api/src/agents/failures/terminal.spec.ts b/packages/api/src/agents/failures/terminal.spec.tsnew file mode 100644index 0000000000..6e3660cae9--- /dev/null+++ b/packages/api/src/agents/failures/terminal.spec.ts@@ -0,0 +1,102 @@+import { createTerminalRunErrorObserver, getUpstreamModelErrorMetadata } from './terminal';++describe('terminal agent-run error logging', () => {+ it('logs stable upstream metadata and deterministic trace correlation', () => {+ const logger = { error: jest.fn() };+ const privateValue = 'PRIVATE-PROVIDER-CONTENT';+ const providerError = Object.assign(new Error(`Provider echoed ${privateValue}`), {+ name: 'InternalServerException',+ code: 'InternalServerException',+ response: {+ status: 500,+ headers: { authorization: privateValue },+ data: { prompt: privateValue },+ },+ });+ const observer = createTerminalRunErrorObserver({+ logger,+ responseMessageId: '78847296-b174-4127-a342-78efa427d4a5',+ source: '[Agent API]',+ });+ observer.modelCallback.handleLLMError(providerError);++ observer.log(new Error('graph failed', { cause: providerError }));++ expect(logger.error).toHaveBeenCalledWith('[Agent API] Upstream model error', {+ type: 'Error',+ status: 500,+ errorCode: 'UPSTREAM_MODEL_ERROR',+ errorOrigin: 'model_provider',+ errorType: '500',+ traceId: '3a90048362ec9a2e717c6b77769b9a54',+ });+ expect(JSON.stringify(logger.error.mock.calls)).not.toContain(privateValue);+ expect(JSON.stringify(logger.error.mock.calls)).not.toContain('InternalServerException');+ });++ it('keeps unrelated terminal failures on the generic safe path', () => {+ const logger = { error: jest.fn() };+ const observer = createTerminalRunErrorObserver({+ logger,+ responseMessageId: 'response-123',+ source: '[Agent API]',+ });+ observer.modelCallback.handleLLMError(new Error('recovered model attempt'));++ observer.log(new Error('checkpoint failed'));++ expect(logger.error).toHaveBeenCalledWith('[Agent API] Error:', { type: 'Error' });+ });++ it('does not log a tracked client cancellation as an upstream failure', () => {+ const logger = { error: jest.fn() };+ const observer = createTerminalRunErrorObserver({ logger, source: '[Agent API]' });+ const controller = new AbortController();+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });+ observer.modelCallback.handleLLMError(abortError);+ controller.abort();++ observer.log(abortError, controller.signal);++ expect(logger.error).not.toHaveBeenCalled();+ });++ it('keeps provider AbortErrors observable while the run signal is live', () => {+ const logger = { error: jest.fn() };+ const observer = createTerminalRunErrorObserver({ logger, source: '[Agent API]' });+ const abortError = Object.assign(new Error('provider aborted'), { name: 'AbortError' });+ observer.modelCallback.handleLLMError(abortError);++ observer.log(abortError, new AbortController().signal);++ expect(logger.error).toHaveBeenCalledWith(+ '[Agent API] Upstream model error',+ expect.objectContaining({ errorCode: 'UPSTREAM_MODEL_ERROR' }),+ );+ });++ it('keeps real provider failures observable when Stop wins the same-tick race', () => {+ const logger = { error: jest.fn() };+ const observer = createTerminalRunErrorObserver({ logger, source: '[Agent API]' });+ const controller = new AbortController();+ const providerError = new Error('provider failed');+ observer.modelCallback.handleLLMError(providerError);+ controller.abort();++ observer.log(providerError, controller.signal);++ expect(logger.error).toHaveBeenCalledWith(+ '[Agent API] Upstream model error',+ expect.objectContaining({ errorCode: 'UPSTREAM_MODEL_ERROR' }),+ );+ });++ it('uses a bounded fallback type and omits unavailable trace correlation', () => {+ expect(getUpstreamModelErrorMetadata(new Error('provider failed'))).toEqual({+ type: 'Error',+ errorCode: 'UPSTREAM_MODEL_ERROR',+ errorOrigin: 'model_provider',+ errorType: '_OTHER',+ });+ });+});diff --git a/packages/api/src/agents/failures/terminal.ts b/packages/api/src/agents/failures/terminal.tsnew file mode 100644index 0000000000..beda93779a--- /dev/null+++ b/packages/api/src/agents/failures/terminal.ts@@ -0,0 +1,81 @@+import type { SafeErrorMetadata } from '../../utils/errors';+import type { ModelErrorTrackerCallback } from './tracker';+import { getSafeErrorMetadata, isAbortError } from '../../utils/errors';+import { traceIdForMessage } from '../../langfuse/trace';+import { createModelErrorTracker } from './tracker';++const UPSTREAM_MODEL_ERROR_CODE = 'UPSTREAM_MODEL_ERROR';+const UPSTREAM_MODEL_ERROR_ORIGIN = 'model_provider';+const UNKNOWN_UPSTREAM_MODEL_ERROR_TYPE = '_OTHER';++export interface UpstreamModelErrorMetadata extends SafeErrorMetadata {+ readonly errorCode: typeof UPSTREAM_MODEL_ERROR_CODE;+ readonly errorOrigin: typeof UPSTREAM_MODEL_ERROR_ORIGIN;+ readonly errorType: string;+ readonly traceId?: string;+}++export interface TerminalRunErrorLogger {+ error(message: string, metadata: SafeErrorMetadata | UpstreamModelErrorMetadata): void;+}++export interface TerminalRunErrorObserver {+ readonly modelCallback: ModelErrorTrackerCallback;+ readonly log: (error: unknown, signal?: AbortSignal) => void;+}++/** A run cancellation requires both host-owned abort state and an abort-shaped rejection. */+export function isAgentRunCancellation(error: unknown, signal?: AbortSignal): boolean {+ return signal?.aborted === true && isAbortError(error);+}++export function getUpstreamModelErrorMetadata(+ error: unknown,+ responseMessageId?: string,+): UpstreamModelErrorMetadata {+ const safeMetadata = getSafeErrorMetadata(error);+ return {+ ...safeMetadata,+ errorCode: UPSTREAM_MODEL_ERROR_CODE,+ errorOrigin: UPSTREAM_MODEL_ERROR_ORIGIN,+ errorType:+ safeMetadata.status != null ? String(safeMetadata.status) : UNKNOWN_UPSTREAM_MODEL_ERROR_TYPE,+ ...(typeof responseMessageId === 'string' && responseMessageId !== ''+ ? { traceId: traceIdForMessage(responseMessageId) }+ : {}),+ };+}++/** Applies one terminal-run log taxonomy to every agent ingress. */+export function createTerminalRunErrorObserver({+ logger,+ responseMessageId,+ source,+ genericMessage = `${source} Error:`,+}: {+ logger: TerminalRunErrorLogger;+ responseMessageId?: string;+ source: string;+ genericMessage?: string;+}): TerminalRunErrorObserver {+ const modelErrorTracker = createModelErrorTracker();+ return Object.freeze({+ modelCallback: modelErrorTracker.callback,+ log(error: unknown, signal?: AbortSignal) {+ if (isAgentRunCancellation(error, signal)) {+ return;+ }++ const upstreamModelError = modelErrorTracker.getUpstreamModelError(error);+ if (upstreamModelError == null) {+ logger.error(genericMessage, getSafeErrorMetadata(error));+ return;+ }++ logger.error(+ `${source} Upstream model error`,+ getUpstreamModelErrorMetadata(upstreamModelError, responseMessageId),+ );+ },+ });+}diff --git a/packages/api/src/agents/failures/tracker.spec.ts b/packages/api/src/agents/failures/tracker.spec.tsnew file mode 100644index 0000000000..41b3ea4cf3--- /dev/null+++ b/packages/api/src/agents/failures/tracker.spec.ts@@ -0,0 +1,143 @@+import { createModelErrorTracker } from './tracker';++describe('createModelErrorTracker', () => {+ it('returns the exact model error that terminates the graph', () => {+ const tracker = createModelErrorTracker();+ const error = new Error('provider failed');++ tracker.callback.handleLLMError(error);++ expect(tracker.getUpstreamModelError(error)).toBe(error);+ });++ it('returns the tracked error through standard cause and aggregate chains', () => {+ const tracker = createModelErrorTracker();+ const providerError = new Error('provider failed');+ tracker.callback.handleLLMError(providerError);++ const terminal = new Error('graph failed', {+ cause: { errors: [new Error('other failure'), providerError] },+ });++ expect(tracker.getUpstreamModelError(terminal)).toBe(providerError);+ });++ it('does not classify a later graph failure after a recovered model attempt', () => {+ const tracker = createModelErrorTracker();+ tracker.callback.handleLLMError(new Error('recovered provider attempt'));++ expect(tracker.getUpstreamModelError(new Error('checkpoint failed'))).toBeNull();+ });++ it('returns the terminal fallback error when multiple attempts were observed', () => {+ const tracker = createModelErrorTracker();+ const primaryError = new Error('primary failed');+ const fallbackError = new Error('fallback failed');+ tracker.callback.handleLLMError(primaryError);+ tracker.callback.handleLLMError(fallbackError);++ expect(tracker.getUpstreamModelError(fallbackError)).toBe(fallbackError);+ });++ it('fails closed for primitive errors and throwing properties', () => {+ const tracker = createModelErrorTracker();+ tracker.callback.handleLLMError('provider failed');+ const hostile = {};+ Object.defineProperties(hostile, {+ cause: {+ get() {+ throw new Error('cause getter failed');+ },+ },+ errors: {+ get() {+ throw new Error('errors getter failed');+ },+ },+ });++ expect(tracker.getUpstreamModelError('provider failed')).toBeNull();+ expect(tracker.getUpstreamModelError(hostile)).toBeNull();+ });++ it('terminates on cyclic error graphs', () => {+ const tracker = createModelErrorTracker();+ const first: Error & { cause?: unknown } = new Error('first');+ const second: Error & { cause?: unknown } = new Error('second');+ first.cause = second;+ second.cause = first;++ expect(tracker.getUpstreamModelError(first)).toBeNull();+ });++ it('caps property reads on an excessively deep cause chain', () => {+ const tracker = createModelErrorTracker();+ let propertyReads = 0;+ let cause: object | undefined;+ for (let depth = 0; depth < 1_000; depth += 1) {+ const nextCause = cause;+ cause = Object.defineProperties(+ {},+ {+ cause: {+ get() {+ propertyReads += 1;+ return nextCause;+ },+ },+ errors: {+ get() {+ propertyReads += 1;+ return undefined;+ },+ },+ },+ );+ }++ expect(tracker.getUpstreamModelError(cause)).toBeNull();+ expect(propertyReads).toBeLessThanOrEqual(64);+ });++ it('caps aggregate entry reads across a hostile wide graph', () => {+ const tracker = createModelErrorTracker();+ let indexReads = 0;+ const errors = new Proxy(+ Array.from({ length: 100_000 }, () => null),+ {+ get(target, property, receiver) {+ if (typeof property === 'string' && /^\d+$/.test(property)) {+ indexReads += 1;+ }+ return Reflect.get(target, property, receiver);+ },+ },+ );++ expect(tracker.getUpstreamModelError({ errors })).toBeNull();+ expect(indexReads).toBeLessThanOrEqual(32);+ });++ it('prioritizes a deep cause chain over aggregate breadth', () => {+ const tracker = createModelErrorTracker();+ const providerError = new Error('provider failed');+ tracker.callback.handleLLMError(providerError);+ const errors = Array.from({ length: 1_000 }, () => new Error('aggregate failure'));++ let terminal: Error & { cause?: unknown } = new Error('terminal');+ const root = terminal;+ for (let depth = 0; depth < 29; depth += 1) {+ const next: Error & { cause?: unknown } = new Error(`wrapper ${depth}`);+ terminal.cause = next;+ terminal = next;+ }+ terminal.cause = providerError;++ expect(+ tracker.getUpstreamModelError({+ errors,+ cause: root,+ }),+ ).toBe(providerError);+ });+});diff --git a/packages/api/src/agents/failures/tracker.ts b/packages/api/src/agents/failures/tracker.tsnew file mode 100644index 0000000000..832112440f--- /dev/null+++ b/packages/api/src/agents/failures/tracker.ts@@ -0,0 +1,116 @@+const MAX_ERROR_GRAPH_NODES = 32;+const MAX_AGGREGATE_ENTRIES_INSPECTED = 32;++type ObjectLike = object | ((...args: never[]) => unknown);++export interface ModelErrorTrackerCallback {+ readonly name: 'librechat-upstream-model-error-tracker';+ readonly awaitHandlers: true;+ readonly handleLLMError: (error: unknown) => void;+}++export interface ModelErrorTracker {+ readonly callback: ModelErrorTrackerCallback;+ readonly getUpstreamModelError: (error: unknown) => object | null;+}++function isObjectLike(value: unknown): value is ObjectLike {+ return (typeof value === 'object' && value !== null) || typeof value === 'function';+}++function readProperty(value: ObjectLike, property: PropertyKey): unknown {+ try {+ return Reflect.get(value, property);+ } catch {+ return undefined;+ }+}++function isArray(value: unknown): value is unknown[] {+ try {+ return Array.isArray(value);+ } catch {+ return false;+ }+}++/**+ * Tracks errors observed at the chat-model callback boundary. Hosts consult+ * the tracker only after the complete graph rejects, so retries and fallbacks+ * that recover never classify an otherwise successful run as upstream.+ */+export function createModelErrorTracker(): ModelErrorTracker {+ const modelErrors = new WeakSet<object>();++ const callback: ModelErrorTrackerCallback = Object.freeze({+ name: 'librechat-upstream-model-error-tracker',+ awaitHandlers: true,+ handleLLMError(error: unknown) {+ if (isObjectLike(error)) {+ modelErrors.add(error);+ }+ },+ });++ const getUpstreamModelError = (error: unknown): object | null => {+ if (!isObjectLike(error)) {+ return null;+ }++ const pendingCauses: ObjectLike[] = [error];+ const pendingAggregates: ObjectLike[] = [];+ const queued = new WeakSet<object>([error]);+ let visitedCount = 0;+ let aggregateEntriesInspected = 0;++ while (+ (pendingCauses.length > 0 || pendingAggregates.length > 0) &&+ visitedCount < MAX_ERROR_GRAPH_NODES+ ) {+ const current = pendingCauses.pop() ?? pendingAggregates.pop();+ if (current == null) {+ continue;+ }+ visitedCount += 1;++ if (modelErrors.has(current)) {+ return current;+ }++ const cause = readProperty(current, 'cause');+ const errors = readProperty(current, 'errors');++ /** Aggregate entries use a secondary worklist, so even a maximally wide+ * aggregate cannot consume the node budget before the standard cause+ * chain is examined. */+ if (isArray(errors)) {+ const length = readProperty(errors, 'length');+ const boundedLength =+ typeof length === 'number' && Number.isSafeInteger(length) && length > 0+ ? Math.min(length, MAX_AGGREGATE_ENTRIES_INSPECTED)+ : 0;+ for (+ let index = boundedLength - 1;+ index >= 0 && aggregateEntriesInspected < MAX_AGGREGATE_ENTRIES_INSPECTED;+ index -= 1+ ) {+ aggregateEntriesInspected += 1;+ const nested = readProperty(errors, index);+ if (isObjectLike(nested) && !queued.has(nested)) {+ queued.add(nested);+ pendingAggregates.push(nested);+ }+ }+ }++ if (isObjectLike(cause) && !queued.has(cause)) {+ queued.add(cause);+ pendingCauses.push(cause);+ }+ }++ return null;+ };++ return Object.freeze({ callback, getUpstreamModelError });+}diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.tsindex 37d3042614..56090134f1 100644--- a/packages/api/src/agents/index.ts+++ b/packages/api/src/agents/index.ts@@ -32,6 +32,7 @@ export * from './lazySubagents';export * from './lazyHistory';export * from './memory';export * from './management';+export * from './failures';export * from './reads';export * from './updates';export * from './mcpIdentity';diff --git a/packages/api/src/agents/remote/host.spec.ts b/packages/api/src/agents/remote/host.spec.tsindex b327a0aaef..49746e4b27 100644--- a/packages/api/src/agents/remote/host.spec.ts+++ b/packages/api/src/agents/remote/host.spec.ts@@ -130,6 +130,38 @@ describe('executeAgentRun', () => {expect(events).toEqual(['rendered', 'settled']);});+ it('passes the canonical aborted signal to execution error handling', async () => {+ let closeListener: (() => void) | undefined;+ const handleExecutionError = jest.fn<string, [unknown, AbortSignal?]>(() => 'handled');+ const abortError = Object.assign(new Error('request aborted'), { name: 'AbortError' });++ await expect(+ executeAgentRun({+ envelope: createEnvelope(),+ runId: 'run-1',+ conversationId: 'conversation-1',+ connection: {+ isClosed: () => false,+ onClose: (listener) => {+ closeListener = listener;+ return () => undefined;+ },+ },+ isPrincipalActive: async () => true,+ execute: async () => {+ closeListener?.();+ throw abortError;+ },+ handleExecutionError,+ }),+ ).resolves.toBe('handled');++ expect(handleExecutionError).toHaveBeenCalledWith(abortError, expect.any(AbortSignal));+ const executionSignal = handleExecutionError.mock.calls[0]?.[1];+ expect(executionSignal).toBeDefined();+ expect(executionSignal?.aborted).toBe(true);+ });+it('settles even when trailing-write registration fails', async () => {const failure = new Error('trailing write registration failed');diff --git a/packages/api/src/agents/remote/host.ts b/packages/api/src/agents/remote/host.tsindex 1522a2a139..a20997f5d1 100644--- a/packages/api/src/agents/remote/host.ts+++ b/packages/api/src/agents/remote/host.ts@@ -15,7 +15,7 @@ export interface ExecuteAgentRunParams<Result> {connection?: AgentExecutionConnection;isPrincipalActive: (userId: string) => Promise<boolean>;execute: (execution: AgentExecutionEnrollment) => Promise<Result>;- handleExecutionError?: (error: unknown) => Result | Promise<Result>;+ handleExecutionError?: (error: unknown, signal?: AbortSignal) => Result | Promise<Result>;beforeSettle?: (execution: AgentExecutionEnrollment,executionError: unknown,@@ -66,7 +66,7 @@ export async function executeAgentRun<Result>({} catch (error) {executionError = error;if (handleExecutionError != null) {- return await handleExecutionError(error);+ return await handleExecutionError(error, execution?.signal);}throw error;} finally {diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.tsindex 21fb29c03a..2c95d4d5e6 100644--- a/packages/api/src/agents/run.ts+++ b/packages/api/src/agents/run.ts@@ -50,6 +50,7 @@ import type { CallbackHandlerMethods } from '@langchain/core/callbacks/base';import type { BaseMessage } from '@librechat/agents/langchain/messages';import type { Callbacks } from '@langchain/core/callbacks/manager';import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent';+import type { ModelErrorTrackerCallback } from '~/agents/failures/tracker';import type { ToolInputValidationError } from '~/agents/toolValidation';import type { ResolvedToolApprovalHook } from '~/agents/hitl/hooks';import type { TerminalSteerHook } from '~/agents/steering/runtime';@@ -986,6 +987,8 @@ type CallbackClientOptions = {fallbacks?: FallbackConfig[];};+type RunModelCallback = ModelBoundChatModelCallback | ModelErrorTrackerCallback;+/*** Installs run-stable callbacks on the model client itself. Subagent child* graphs intentionally replace invocation callbacks with their own event@@ -994,7 +997,7 @@ type CallbackClientOptions = {*/function withModelCallbacks<T extends object>(options: T,- modelCallbacks: readonly ModelBoundChatModelCallback[] | undefined,+ modelCallbacks: readonly RunModelCallback[] | undefined,): T {if (!modelCallbacks?.length) {return options;@@ -1640,8 +1643,8 @@ export async function createRun({compactionSemanticIndex?: CompactionSemanticIndex;/** Cross-run summary from formatAgentMessages, forwarded to AgentContext */initialSummary?: { text: string; tokenCount: number };- /** Model-level guards inherited by root, summary, fallback, and subagent clients. */- modelCallbacks?: readonly ModelBoundChatModelCallback[];+ /** Model callbacks inherited by root, summary, fallback, and subagent clients. */+ modelCallbacks?: readonly RunModelCallback[];/** Calibration ratio from previous run's contextMeta, seeds the pruner EMA */calibrationRatio?: number;/**diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.tsindex 487369aa86..7f2e4b3573 100644--- a/packages/api/src/agents/testHook.ts+++ b/packages/api/src/agents/testHook.ts@@ -1,7 +1,6 @@import { logger } from '@librechat/data-schemas';import type { BaseMessage } from '@librechat/agents/langchain/messages';import type { Run, IState } from '@librechat/agents';-import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent';/*** Context handed to a test run hook so it can shape fake-model behavior from@@ -17,7 +16,8 @@ export interface TestRunHookContext {*/conversationId?: string;agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>;- modelCallbacks?: readonly ModelBoundChatModelCallback[];+ /** Opaque model-client handlers forwarded to the fake model by the e2e harness. */+ modelCallbacks?: readonly object[];}export type TestRunHook = (run: Run<IState>, context: TestRunHookContext) => void;