Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📬 fix: Persist Background Results Before Parent Rows (#16101)

Danny Avila8a3ab439659e
diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js
index 6fac81d976..b02664babc 100644
--- a/api/server/controllers/agents/__tests__/openai.spec.js
+++ b/api/server/controllers/agents/__tests__/openai.spec.js
@@ -1355,7 +1355,10 @@ describe('OpenAIChatCompletionController', () => {
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
- req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
+ req.config.endpoints.agents.backgroundTasks = {
+ ordinaryToolCancellation: true,
+ completionResultMaxChars: 4096,
+ };
await OpenAIChatCompletionController(req, res);
const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1);
@@ -1384,6 +1387,7 @@ describe('OpenAIChatCompletionController', () => {
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
+ expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal);
expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId);
const effectiveSignal = new AbortController().signal;
diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js
index 45d00b5684..70f0e93754 100644
--- a/api/server/controllers/agents/__tests__/responses.unit.spec.js
+++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js
@@ -2065,7 +2065,10 @@ describe('createResponse controller', () => {
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
- req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
+ req.config.endpoints.agents.backgroundTasks = {
+ ordinaryToolCancellation: true,
+ completionResultMaxChars: 4096,
+ };
req.body.stream = stream;
await createResponse(req, res);
@@ -2095,6 +2098,7 @@ describe('createResponse controller', () => {
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
+ expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal);
expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId);
const effectiveSignal = new AbortController().signal;
diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js
index 7bcbe4b9fa..65e2230594 100644
--- a/api/server/controllers/agents/openai.js
+++ b/api/server/controllers/agents/openai.js
@@ -490,6 +490,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
const allowedProviders = new Set(agentsEConfig?.allowedProviders);
const ordinaryToolCancellationEnabled =
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
+ const backgroundCompletionResultMaxChars =
+ agentsEConfig?.backgroundTasks?.completionResultMaxChars;
// Create tool loader
const loadTools = createToolLoader({ req, res, signal: execution.signal });
@@ -840,6 +842,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
runSignal: execution.signal,
foregroundRunId: responseId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
+ backgroundCompletionResultMaxChars,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,
diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js
index 24e668576b..1b4772ffd0 100644
--- a/api/server/controllers/agents/responses.js
+++ b/api/server/controllers/agents/responses.js
@@ -732,6 +732,8 @@ const executeResponse = async (envelope, { req, res }) => {
const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents];
const ordinaryToolCancellationEnabled =
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
+ const backgroundCompletionResultMaxChars =
+ agentsEConfig?.backgroundTasks?.completionResultMaxChars;
const previousMessages = request.previous_response_id
? await loadPreviousMessages(request.previous_response_id, principal.userId)
: [];
@@ -1194,6 +1196,7 @@ const executeResponse = async (envelope, { req, res }) => {
runSignal: execution.signal,
foregroundRunId: responseId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
+ backgroundCompletionResultMaxChars,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,
@@ -1424,6 +1427,7 @@ const executeResponse = async (envelope, { req, res }) => {
runSignal: execution.signal,
foregroundRunId: responseId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
+ backgroundCompletionResultMaxChars,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index 66926c2991..1854c70e84 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -86,6 +86,9 @@ module.exports = {
requeueAgentTrigger: service.requeue,
retireAgentTrigger: service.retire,
renewAgentTriggerProducerLease: service.renewProducerLease,
+ persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
+ getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
+ releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
drainAgentTriggerDeliveriesForUser: service.drainUser,
prepareAgentTriggerUserPurge: service.prepareUserPurge,
cancelAgentTriggerUserPurge: service.cancelUserPurge,
diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js
index 6fc0996e62..8ca1cae6ef 100644
--- a/api/server/services/Endpoints/agents/backgroundCompletion.js
+++ b/api/server/services/Endpoints/agents/backgroundCompletion.js
@@ -2,9 +2,13 @@ const {
createBackgroundToolCompletionWakeupHandler,
createBackgroundToolDeadClaimRecovery,
createBackgroundToolResultHandler,
+ claimBackgroundToolResult: claimResult,
} = require('@librechat/api');
const {
enqueueAgentTrigger,
+ persistAgentBackgroundToolResult,
+ getAgentBackgroundToolResultClaim,
+ releaseAgentBackgroundToolResultClaims,
renewAgentTriggerProducerLease,
retireAgentTrigger,
} = require('../../Agents/triggers');
@@ -13,12 +17,17 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
enqueueAgentTrigger,
retireAgentTrigger,
renewAgentTriggerProducerLease,
+ (deliveryKey, sourceId, result) =>
+ persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
);
function createBackgroundToolResultPersistence({ req, updateToolCallResult }) {
return createBackgroundToolResultHandler({ req, updateToolCallResult });
}
+const claimBackgroundToolResult = (db, input) =>
+ claimResult(db, getAgentBackgroundToolResultClaim, input);
+
function createDeadBackgroundToolClaimRecovery(
releaseBackgroundToolResultClaims,
getGenerationJob,
@@ -29,11 +38,13 @@ function createDeadBackgroundToolClaimRecovery(
releaseBackgroundToolResultClaims,
getGenerationJob,
fenceGenerationClaim,
+ releaseAgentBackgroundToolResultClaims,
);
}
module.exports = {
preregisterBackgroundToolCompletion,
createBackgroundToolResultPersistence,
+ claimBackgroundToolResult,
createDeadBackgroundToolClaimRecovery,
};
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index ee8ebb207d..875d085951 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -95,6 +95,7 @@ const subagentThreadTaskStore = require('./subagentThreadStore');
const {
preregisterBackgroundToolCompletion,
createBackgroundToolResultPersistence,
+ claimBackgroundToolResult,
createDeadBackgroundToolClaimRecovery,
} = require('./backgroundCompletion');
const { logViolation } = require('~/cache');
@@ -215,6 +216,8 @@ const initializeClientWithProvider = async ({
const ordinaryToolCancellationEnabled =
appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.ordinaryToolCancellation ===
true;
+ const backgroundCompletionResultMaxChars =
+ appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.completionResultMaxChars;
/** The normal controller resolves this once for timestamp anchoring. Reuse
* that trusted document for child-thread execution policy; resume and direct
* callers fall back to the same owner-scoped lookup. */
@@ -440,6 +443,7 @@ const initializeClientWithProvider = async ({
runSignal: signal,
foregroundRunId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
+ backgroundCompletionResultMaxChars,
loadTools: async (
toolNames,
agentId,
@@ -515,7 +519,7 @@ const initializeClientWithProvider = async ({
req,
updateToolCallResult: db.updateToolCallResult,
}),
- claim: db.claimBackgroundToolResults,
+ claim: (input) => claimBackgroundToolResult(db, input),
recoverDeadClaim: createDeadBackgroundToolClaimRecovery(
db.releaseBackgroundToolResultClaims,
(conversationId) => GenerationJobManager.getJob(conversationId),
diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js
index b01af98760..fe65c2493a 100644
--- a/api/server/services/Endpoints/agents/initialize.spec.js
+++ b/api/server/services/Endpoints/agents/initialize.spec.js
@@ -408,10 +408,11 @@ describe('initializeClient — processAgent ACL gate', () => {
endpointOption: makeEndpointOption(),
});
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(false);
+ expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBeUndefined();
const enabledReq = makeReq();
enabledReq.config.endpoints.agents = {
- backgroundTasks: { ordinaryToolCancellation: true },
+ backgroundTasks: { ordinaryToolCancellation: true, completionResultMaxChars: 4096 },
};
await initializeClient({
req: enabledReq,
@@ -420,6 +421,7 @@ describe('initializeClient — processAgent ACL gate', () => {
endpointOption: makeEndpointOption(),
});
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(true);
+ expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
});
it('propagates an expected-MCP-tools failure from the runtime tool loader', async () => {
diff --git a/packages/api/src/agents/backgroundClaims.spec.ts b/packages/api/src/agents/backgroundClaims.spec.ts
new file mode 100644
index 0000000000..f0bf556e90
--- /dev/null
+++ b/packages/api/src/agents/backgroundClaims.spec.ts
@@ -0,0 +1,82 @@
+import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas';
+import { claimBackgroundToolResult } from './backgroundClaims';
+
+describe('manual background receipt arbitration', () => {
+ const input = {
+ userId: 'user',
+ conversationId: 'conversation',
+ taskId: 'task',
+ kind: 'manual' as const,
+ claimId: 'poll',
+ };
+ const receipt = { kind: 'wakeup' as const, claimId: 'delivery', claimedAt: new Date() };
+ function fixture() {
+ const methods = {
+ claimBackgroundToolResults: jest
+ .fn<
+ ReturnType<MessageMethods['claimBackgroundToolResults']>,
+ Parameters<MessageMethods['claimBackgroundToolResults']>
+ >()
+ .mockResolvedValue({
+ status: 'acquired',
+ messageId: 'recovered-parent',
+ results: [
+ {
+ taskId: 'task',
+ toolCallId: 'call',
+ toolName: 'tool',
+ status: 'completed',
+ output: 'done',
+ },
+ ],
+ }),
+ releaseBackgroundToolResultClaims: jest.fn(async () => true),
+ };
+ const lookup = jest
+ .fn<
+ ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>,
+ Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>
+ >()
+ .mockResolvedValue(null);
+ return { methods, lookup };
+ }
+ it('uses the recovered parent identity when the process-local registry is gone', async () => {
+ const { methods, lookup } = fixture();
+ lookup.mockResolvedValue(receipt);
+ await expect(claimBackgroundToolResult(methods, lookup, input)).resolves.toEqual({
+ status: 'claimed',
+ claim: receipt,
+ messageId: 'recovered-parent',
+ });
+ expect(lookup).toHaveBeenCalledWith(
+ expect.objectContaining({ parentMessageId: 'recovered-parent' }),
+ );
+ expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
+ expect.objectContaining({ messageId: 'recovered-parent', taskIds: ['task'] }),
+ );
+ });
+ it('releases a speculative message claim if receipt lookup fails', async () => {
+ const { methods, lookup } = fixture();
+ lookup.mockRejectedValue(new Error('database unavailable'));
+ await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow(
+ 'database unavailable',
+ );
+ expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledTimes(1);
+ });
+ it('does not report arbitration as complete if rollback fails', async () => {
+ const { methods, lookup } = fixture();
+ lookup.mockResolvedValue(receipt);
+ methods.releaseBackgroundToolResultClaims.mockResolvedValue(false);
+ await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow(
+ 'could not be released',
+ );
+ });
+ it('does not claim the projection when its known receipt already has an owner', async () => {
+ const { methods, lookup } = fixture();
+ lookup.mockResolvedValue(receipt);
+ await expect(
+ claimBackgroundToolResult(methods, lookup, { ...input, messageId: 'parent' }),
+ ).resolves.toMatchObject({ status: 'claimed', messageId: 'parent' });
+ expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/api/src/agents/backgroundClaims.ts b/packages/api/src/agents/backgroundClaims.ts
new file mode 100644
index 0000000000..6f92bca763
--- /dev/null
+++ b/packages/api/src/agents/backgroundClaims.ts
@@ -0,0 +1,58 @@
+import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas';
+import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup';
+
+/** Reconciles manual polling with the independent automatic-delivery receipt,
+ * including polls reconstructed after the process-local registry was lost. */
+export async function claimBackgroundToolResult(
+ methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,
+ getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],
+ input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],
+): ReturnType<MessageMethods['claimBackgroundToolResults']> {
+ const lookup = (messageId: string) =>
+ getReceiptClaim({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId: input.userId,
+ conversationId: input.conversationId,
+ parentMessageId: messageId,
+ taskId: input.taskId,
+ });
+ if (input.messageId != null) {
+ const receiptClaim = await lookup(input.messageId);
+ if (receiptClaim != null && receiptClaim.claimId !== input.claimId) {
+ return { status: 'claimed', claim: receiptClaim, messageId: input.messageId };
+ }
+ }
+ const messageClaim = await methods.claimBackgroundToolResults(input);
+ if (messageClaim.status !== 'acquired') {
+ return messageClaim;
+ }
+ const messageId = input.messageId ?? messageClaim.messageId;
+ if (messageId == null) {
+ throw new Error('The background result claim has no parent message identity');
+ }
+ const release = async () => {
+ const released = await methods.releaseBackgroundToolResultClaims({
+ userId: input.userId,
+ conversationId: input.conversationId,
+ messageId,
+ taskIds: messageClaim.results.map((result) => result.taskId),
+ kind: input.kind,
+ claimId: input.claimId,
+ });
+ if (!released) {
+ throw new Error('The background result claim could not be released');
+ }
+ };
+ let receiptClaim: Awaited<ReturnType<typeof getReceiptClaim>>;
+ try {
+ receiptClaim = await lookup(messageId);
+ } catch (error) {
+ await release();
+ throw error;
+ }
+ if (receiptClaim == null || receiptClaim.claimId === input.claimId) {
+ return messageClaim;
+ }
+ await release();
+ return { status: 'claimed', claim: receiptClaim, messageId };
+}
diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts
index 08b84b0b80..8ef1d74cca 100644
--- a/packages/api/src/agents/backgroundCompletion.ts
+++ b/packages/api/src/agents/backgroundCompletion.ts
@@ -31,6 +31,13 @@ export interface BackgroundToolWakeupRetireOptions {
export interface BackgroundToolWakeupAdmission {
/** Renews durable proof that the process-local executor still owns work. */
renew: () => Promise<boolean>;
+ /** Persists terminal output on the pre-admitted delivery before the parent
+ * message projection exists. */
+ persistResult?: (result: {
+ status: 'completed' | 'error' | 'cancelled';
+ output: string;
+ settledAt: Date;
+ }) => Promise<boolean>;
/** Retires a delivery whose terminal result can no longer be made durable.
* Manual polling requires an atomic unclaimed-only transition: once a
* resolver owns a lease, its prepared continuation cannot be cancelled. */
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
index 19a4a9560d..90f226f634 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
@@ -1,4 +1,4 @@
-import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
+import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
import {
@@ -91,6 +91,10 @@ function resolverMethods() {
leaseUntil: new Date(NOW + 30_000),
}),
),
+ getAgentBackgroundToolResult: jest.fn(async () => null),
+ getAgentBackgroundToolResultClaim: jest.fn(async () => null),
+ claimAgentBackgroundToolResults: jest.fn(async () => ({ status: 'not_ready' as const })),
+ releaseAgentBackgroundToolResultClaims: jest.fn(async () => true),
},
};
}
@@ -111,7 +115,13 @@ describe('background tool completion wakeups', () => {
>(async () => ({ deliveryKey: 'delivery-key-1' }));
const retire = jest.fn(async () => true);
const renew = jest.fn(async () => true);
- const notify = createBackgroundToolCompletionWakeupHandler(enqueue, retire, renew);
+ const persistResult = jest.fn(async () => true);
+ const notify = createBackgroundToolCompletionWakeupHandler(
+ enqueue,
+ retire,
+ renew,
+ persistResult,
+ );
const admission = await notify(registration());
expect(admission).not.toBe(false);
@@ -134,7 +144,7 @@ describe('background tool completion wakeups', () => {
expect(options).toEqual({
orderingKey: 'background-tool-completion:conversation-1:task-1',
availableAt: new Date(NOW + 250),
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: new Date(NOW + 30_000),
});
if (admission !== false) {
@@ -145,6 +155,20 @@ describe('background tool completion wakeups', () => {
'background-tool-completion',
new Date(NOW + 30_000),
);
+ if (admission !== false) {
+ await expect(
+ admission.persistResult?.({
+ status: 'completed',
+ output: 'done',
+ settledAt: new Date(NOW),
+ }),
+ ).resolves.toBe(true);
+ }
+ expect(persistResult).toHaveBeenCalledWith('delivery-key-1', 'background-tool-completion', {
+ status: 'completed',
+ output: 'done',
+ settledAt: new Date(NOW),
+ });
if (admission !== false) {
await expect(admission.retire('result unavailable')).resolves.toBe(true);
}
@@ -180,6 +204,7 @@ describe('background tool completion wakeups', () => {
it('retires the batch-root delivery before releasing all of its sibling claims', async () => {
const retire = jest.fn(async () => true);
const release = jest.fn(async () => true);
+ const releaseReceipts = jest.fn(async () => true);
const getGenerationJob = jest.fn(async () => ({ status: 'complete' }));
const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
const recover = createBackgroundToolDeadClaimRecovery(
@@ -187,6 +212,7 @@ describe('background tool completion wakeups', () => {
release,
getGenerationJob,
fenceGenerationClaim,
+ releaseReceipts,
);
await expect(
@@ -218,6 +244,13 @@ describe('background tool completion wakeups', () => {
kind: 'wakeup',
claimId: 'batch-root-delivery',
});
+ expect(releaseReceipts).toHaveBeenCalledWith({
+ sourceId: 'background-tool-completion',
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ claimId: 'batch-root-delivery',
+ });
});
it('keeps a manual result claim while its owning generation is active', async () => {
@@ -428,6 +461,120 @@ describe('background tool completion wakeups', () => {
);
});
+ it('continues from the independent delivery receipt before the parent projection lands', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
+ methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
+ status: 'acquired',
+ results: [
+ {
+ taskId: 'task-1',
+ toolCallId: 'call-1',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'independently durable',
+ },
+ ],
+ } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
+
+ expect(prepared).toMatchObject({ status: 'ready', parentMessageId: 'response-2' });
+ expect(prepared?.status === 'ready' && prepared.input).toContain('independently durable');
+ expect(methods.claimAgentBackgroundToolResults).toHaveBeenCalledWith({
+ deliveryKey: 'delivery-1',
+ sourceId: 'background-tool-completion',
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ agentId: 'agent_parent_1',
+ claimId: 'delivery-1',
+ limit: 1,
+ });
+ expect(methods.claimBackgroundToolResults).toHaveBeenCalledWith(
+ expect.objectContaining({ taskId: 'task-1', kind: 'wakeup', claimId: 'delivery-1' }),
+ );
+ });
+
+ it('honors a manual message claim before an independent receipt', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).resolves.toEqual({
+ status: 'settled',
+ });
+ expect(methods.claimAgentBackgroundToolResults).not.toHaveBeenCalled();
+ expect(methods.getAgentBackgroundToolResult).not.toHaveBeenCalled();
+ });
+
+ it('does not settle a resumed receipt owner while a speculative manual message claim yields', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] });
+ methods.getAgentBackgroundToolResultClaim.mockResolvedValueOnce({
+ kind: 'wakeup',
+ claimId: 'delivery-1',
+ claimedAt: new Date(NOW),
+ } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ deferWithoutAttempt: true,
+ },
+ );
+ expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
+ expect.objectContaining({ claimId: 'delivery-1' }),
+ );
+ expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();
+ });
+
+ it('retries after yielding to a racing manual claim so mutual yielding cannot lose output', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults
+ .mockResolvedValueOnce({ status: 'not_ready' } as never)
+ .mockResolvedValueOnce({
+ status: 'claimed',
+ claim: { kind: 'manual', claimId: 'poll-1' },
+ } as never);
+ methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
+ status: 'acquired',
+ results: [
+ {
+ taskId: 'task-1',
+ toolCallId: 'call-1',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'durable',
+ },
+ ],
+ } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ deferWithoutAttempt: true,
+ },
+ );
+ expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
+ expect.objectContaining({ claimId: 'delivery-1', parentMessageId: 'response-1' }),
+ );
+ });
+
it('shares one bounded input budget across a full sibling batch', async () => {
const { methods } = resolverMethods();
const results = Array.from({ length: 8 }, (_, index) => ({
@@ -468,7 +615,10 @@ describe('background tool completion wakeups', () => {
await prepared.releaseOnDefiniteFailure?.();
}
expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
- expect.objectContaining({ taskIds: ['task-1', 'task-2'], claimId: 'delivery-1' }),
+ expect.objectContaining({ claimId: 'delivery-1', kind: 'wakeup' }),
+ );
+ expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
+ expect.not.objectContaining({ taskIds: expect.anything() }),
);
});
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts
index bdb2686940..ee4c8e992d 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts
@@ -1,8 +1,9 @@
import { randomUUID } from 'node:crypto';
import { isEphemeralAgentId } from 'librechat-data-provider';
-import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
+import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
import type {
AgentTriggerProducerLeaseStatus,
+ AgentTriggerDeliveryMethods,
BackgroundToolResultClaim,
ConversationMethods,
IMessage,
@@ -51,6 +52,16 @@ export type RenewBackgroundToolCompletionProducerLease = (
leaseUntil: Date,
) => Promise<boolean>;
+export type PersistBackgroundToolCompletionResult = (
+ deliveryKey: string,
+ sourceId: string,
+ result: {
+ status: 'completed' | 'error' | 'cancelled';
+ output: string;
+ settledAt: Date;
+ },
+) => Promise<boolean>;
+
type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
Pick<MessageMethods, 'getMessages'> & {
claimBackgroundToolResults(params: {
@@ -76,6 +87,14 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
sourceId: string;
now: Date;
}): Promise<AgentTriggerProducerLeaseStatus>;
+ getAgentBackgroundToolResult?(params: { deliveryKey: string; sourceId: string }): Promise<{
+ status: 'completed' | 'error' | 'cancelled';
+ output: string;
+ settledAt: Date;
+ } | null>;
+ claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
+ getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
+ releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
};
interface GenerationState {
@@ -326,8 +345,33 @@ export function createBackgroundToolCompletionWakeupResolver({
agentId: envelope.target.agentId,
kind: 'wakeup',
claimId: context.idempotencyKey,
+ limit: 1,
});
if (claim.status === 'claimed') {
+ const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ taskId: registration.taskId,
+ });
+ if (receiptOwner?.claimId === context.idempotencyKey) {
+ const released = await methods.releaseAgentBackgroundToolResultClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ claimId: context.idempotencyKey,
+ });
+ if (released === false)
+ throw new Error('Background receipt claim release was not confirmed');
+ throw executionError('Background result ownership is being reconciled.', {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+ }
return { status: 'settled' };
}
if (claim.status === 'outcome_unknown') {
@@ -336,47 +380,17 @@ export function createBackgroundToolCompletionWakeupResolver({
retryable: false,
});
}
- if (claim.status !== 'acquired') {
- let producerLease: AgentTriggerProducerLeaseStatus;
- try {
- producerLease = await methods.getAgentTriggerDeliveryProducerLease({
- deliveryKey: context.idempotencyKey,
- sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
- now: new Date(),
- });
- } catch (error) {
- throw executionError(
- `Background tool producer liveness is temporarily unavailable: ${
- error instanceof Error ? error.message : String(error)
- }`,
- { code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
- );
- }
- if (producerLease.status === 'expired') {
- throw executionError('The process-local background tool executor was lost.', {
- code: 'BACKGROUND_TOOL_PRODUCER_LOST',
- retryable: false,
- });
- }
- /** A live lease proves the invocation or its durable persistence retry
- * still has an owner. Missing remains defer-only for compatibility with
- * completion rows admitted before producer leases existed. */
- throw executionError('The background tool result is not durable yet.', {
- code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
- retryable: true,
- status: 409,
- retryAfter: '1',
- deferWithoutAttempt: true,
+ if (claim.status === 'acquired') {
+ const taskIds = claim.results.map((result) => result.taskId);
+ const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ taskId: registration.taskId,
});
- }
- const taskIds = claim.results.map((result) => result.taskId);
- const input = buildWakeupInput(claim.results);
- return {
- status: 'ready',
- parentMessageId,
- input,
- releaseOnDefiniteFailure: async () => {
- await methods.releaseBackgroundToolResultClaims({
+ if (receiptOwner != null && receiptOwner.claimId !== context.idempotencyKey) {
+ const released = await methods.releaseBackgroundToolResultClaims({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
@@ -384,8 +398,153 @@ export function createBackgroundToolCompletionWakeupResolver({
kind: 'wakeup',
claimId: context.idempotencyKey,
});
- },
- };
+ if (!released) throw new Error('Background projection claim release was not confirmed');
+ return { status: 'settled' };
+ }
+ const input = buildWakeupInput(claim.results);
+ return {
+ status: 'ready',
+ parentMessageId,
+ input,
+ releaseOnDefiniteFailure: async () => {
+ const released = await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ });
+ if (!released) throw new Error('Background projection claim release was not confirmed');
+ const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ claimId: context.idempotencyKey,
+ });
+ if (receiptReleased === false)
+ throw new Error('Background receipt claim release was not confirmed');
+ },
+ };
+ }
+ const receiptClaim = await methods.claimAgentBackgroundToolResults?.({
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ agentId: envelope.target.agentId,
+ claimId: context.idempotencyKey,
+ limit: 1,
+ });
+ if (receiptClaim?.status === 'claimed') {
+ return { status: 'settled' };
+ }
+ if (receiptClaim?.status === 'acquired') {
+ /** Reconcile with a parent projection that may have appeared while the
+ * receipt CAS was in flight. A manual poll that already owns the message
+ * wins; otherwise stamp this same automatic owner before dispatch. */
+ const projectedClaim = await methods.claimBackgroundToolResults({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskId: registration.taskId,
+ agentId: envelope.target.agentId,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ limit: 1,
+ });
+ if (projectedClaim.status === 'claimed') {
+ const released = await methods.releaseAgentBackgroundToolResultClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ claimId: context.idempotencyKey,
+ });
+ if (released === false)
+ throw new Error('Background receipt claim release was not confirmed');
+ throw executionError('Background result ownership is being reconciled.', {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+ }
+ return {
+ status: 'ready',
+ parentMessageId,
+ input: buildWakeupInput(receiptClaim.results),
+ releaseOnDefiniteFailure: async () => {
+ const projectionReleased = await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ });
+ if (projectedClaim.status === 'acquired' && !projectionReleased) {
+ throw new Error('Background projection claim release was not confirmed');
+ }
+ const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ claimId: context.idempotencyKey,
+ });
+ if (receiptReleased === false)
+ throw new Error('Background receipt claim release was not confirmed');
+ },
+ };
+ }
+ const receipt =
+ methods.claimAgentBackgroundToolResults == null
+ ? await methods.getAgentBackgroundToolResult?.({
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ })
+ : null;
+ if (receipt != null) {
+ return {
+ status: 'ready',
+ parentMessageId,
+ input: buildWakeupInput([
+ { ...registration, status: receipt.status, output: receipt.output },
+ ]),
+ };
+ }
+ let producerLease: AgentTriggerProducerLeaseStatus;
+ try {
+ producerLease = await methods.getAgentTriggerDeliveryProducerLease({
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ now: new Date(),
+ });
+ } catch (error) {
+ throw executionError(
+ `Background tool producer liveness is temporarily unavailable: ${
+ error instanceof Error ? error.message : String(error)
+ }`,
+ { code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
+ );
+ }
+ if (producerLease.status === 'expired') {
+ throw executionError('The process-local background tool executor was lost.', {
+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',
+ retryable: false,
+ });
+ }
+ /** A live lease proves the invocation or its durable persistence retry
+ * still has an owner. Missing remains defer-only for compatibility with
+ * completion rows admitted before producer leases existed. */
+ throw executionError('The background tool result is not durable yet.', {
+ code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
+ retryable: true,
+ status: 409,
+ retryAfter: '1',
+ deferWithoutAttempt: true,
+ });
};
}
@@ -394,6 +553,7 @@ export function createBackgroundToolCompletionWakeupHandler(
enqueue: EnqueueBackgroundToolCompletion,
retire: RetireBackgroundToolCompletion,
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
+ persistResult?: PersistBackgroundToolCompletionResult,
): (
registration: BackgroundToolWakeupRegistration,
) => Promise<BackgroundToolWakeupAdmission | false> {
@@ -437,7 +597,7 @@ export function createBackgroundToolCompletionWakeupHandler(
availableAt: new Date(
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
),
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
});
return {
@@ -447,6 +607,12 @@ export function createBackgroundToolCompletionWakeupHandler(
BACKGROUND_TOOL_COMPLETION_SOURCE,
new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
),
+ ...(persistResult == null
+ ? {}
+ : {
+ persistResult: (result) =>
+ persistResult(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, result),
+ }),
retire: (reason, options) =>
options == null
? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason)
@@ -469,6 +635,7 @@ export function createBackgroundToolDeadClaimRecovery(
conversationId: string;
claimId: string;
}) => Promise<'fenced' | 'started' | 'unavailable'>,
+ releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
): BackgroundToolDeadClaimRecovery {
return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
if (kind === 'manual') {
@@ -518,12 +685,24 @@ export function createBackgroundToolDeadClaimRecovery(
if (await claimGenerationIsActive()) {
return false;
}
- return releaseClaims({
+ const released = await releaseClaims({
userId,
conversationId,
messageId,
kind: 'wakeup',
claimId,
});
+ if (!released) {
+ return false;
+ }
+ return (
+ (await releaseReceiptClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId,
+ parentMessageId: messageId,
+ claimId,
+ })) ?? true
+ );
};
}
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index ef1202bf4a..645a473cf9 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas';
import { tool as structuredTool } from '@librechat/agents/langchain/tools';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
+import type { BackgroundToolWakeupAdmission } from './backgroundCompletion';
import type { ToolExecuteOptions } from './handlers';
import {
BACKGROUND_TASK_ABORT_GRACE_MS,
@@ -147,9 +148,13 @@ describe('createToolExecuteHandler — background tool calls', () => {
it('pre-registers an ordinary completion before invoke and persists its terminal receipt', async () => {
const events: string[] = [];
const retire = jest.fn(async () => true);
+ const persistResult = jest.fn(async () => {
+ events.push('receipt');
+ return true;
+ });
const preregister = jest.fn(async () => {
events.push('preregister');
- return { renew: jest.fn(async () => true), retire };
+ return { renew: jest.fn(async () => true), persistResult, retire };
});
const persist = jest.fn(async () => {
events.push('persist');
@@ -198,7 +203,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
);
expect(JSON.parse(dispatch.content).message).toContain('host will resume you');
await flushMicrotasks();
- expect(events).toEqual(['preregister', 'invoke', 'persist']);
+ expect(events).toEqual(['preregister', 'invoke', 'receipt', 'persist']);
expect(preregister).toHaveBeenCalledWith(
expect.objectContaining({
toolCallId: 'call-wakeup',
@@ -217,6 +222,9 @@ describe('createToolExecuteHandler — background tool calls', () => {
}),
}),
);
+ expect(persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'completed', output: 'durable result' }),
+ );
expect(retire).not.toHaveBeenCalled();
});
@@ -264,6 +272,55 @@ describe('createToolExecuteHandler — background tool calls', () => {
);
});
+ it.each(['acknowledged', 'ambiguous'])(
+ 'keeps delivery live for an %s receipt without a projection',
+ async (ack) => {
+ const retire = jest.fn(async () => true);
+ const persistResult = jest.fn(async () => {
+ if (ack === 'ambiguous') throw new Error('receipt committed but acknowledgement lost');
+ return true;
+ });
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(async () => ({ content: 'available before parent row' })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ persistResult,
+ retire,
+ })),
+ persist: jest.fn(async () => false),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: `call-independent-receipt-${ack}`,
+ name: tool.name,
+ args: { q: 'receipt', run_in_background: true },
+ stepId: 'step-independent-receipt',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: `response-independent-receipt-${ack}` },
+ });
+ await flushMicrotasks();
+
+ expect(persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'completed', output: 'available before parent row' }),
+ );
+ expect(retire).not.toHaveBeenCalled();
+ },
+ );
+
it('keeps polling guidance when the completion adapter skips registration', async () => {
const tool = makeSearchTool({ calls: 0 });
const preregister = jest.fn(async () => false as const);
@@ -2570,6 +2627,84 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
expect(poll[0].artifact).toEqual(CODE_ARTIFACT);
});
+ it('does not publish an output-only receipt when generated files miss their message anchor', async () => {
+ const state: CodeToolState = { calls: 0 };
+ const retire = jest.fn(async () => true);
+ const persistResult = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
+ persistBackgroundCodeResult: async () => ({
+ deliveryReady: false,
+ attachments: [{ filename: 'plot.png' }],
+ }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ persistResult,
+ retire,
+ })),
+ persist: jest.fn(async () => true),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [codeCall({ id: 'call_code_missing_anchor' })],
+ agentId: 'a',
+ configurable: buildConfig(['execute_code']),
+ metadata: { thread_id: 'exec_convo_code_missing_anchor', run_id: 'msg-missing-anchor' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(persistResult).not.toHaveBeenCalled();
+ expect(retire).toHaveBeenCalledWith('background code result was not persisted', undefined);
+ });
+
+ it('uses the configured durable completion result size', async () => {
+ const persistResult: jest.MockedFunction<
+ NonNullable<BackgroundToolWakeupAdmission['persistResult']>
+ > = jest.fn(
+ async (_result: Parameters<NonNullable<BackgroundToolWakeupAdmission['persistResult']>>[0]) =>
+ true,
+ );
+ const tool = {
+ name: 'execute_code',
+ description: 'run code',
+ schema: z.object({ lang: z.string(), code: z.string() }),
+ invoke: jest.fn(async () => ({ content: '0123456789'.repeat(10), artifact: CODE_ARTIFACT })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundCompletionResultMaxChars: 32,
+ persistBackgroundCodeResult: async () => ({ deliveryReady: true, attachments: [] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ persistResult,
+ retire: jest.fn(async () => true),
+ })),
+ persist: jest.fn(async () => true),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [codeCall({ id: 'call-sized-receipt' })],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-sized-receipt' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'completed', output: expect.any(String) }),
+ );
+ expect(persistResult.mock.calls[0][0].output).toHaveLength(32);
+ });
+
it('makes a completion-time generated-file policy rejection terminal across polls', async () => {
const protectedValue = 'PROTECTED-GENERATED-FILE-BYTES';
const blockedArtifact = {
@@ -2596,12 +2731,17 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
});
const toolEndCallback = jest.fn();
const retire = jest.fn(async () => true);
+ const persistResult = jest.fn(async () => true);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [codeTool] }),
toolEndCallback,
persistBackgroundCodeResult,
backgroundToolCompletion: {
- preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire,
+ persistResult,
+ })),
persist: jest.fn(async () => true),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
@@ -2621,7 +2761,10 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
const taskId = JSON.parse(dispatch[0].content).background_task_id;
await flushMicrotasks();
await flushMicrotasks();
- expect(retire).toHaveBeenCalledWith('background code result persistence failed', undefined);
+ expect(persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', output: MODEL_BOUND_FILE_CONTENT_BLOCK }),
+ );
+ expect(retire).not.toHaveBeenCalled();
await flushMicrotasks();
for (const pollId of ['call_policy_poll_1', 'call_policy_poll_2']) {
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index 155f20bd18..44d8f50bc8 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -3,6 +3,7 @@ import { Types } from 'mongoose';
import { GraphEvents, Constants, ToolEndHandler } from '@librechat/agents';
import { logger, normalizeSkillFrontmatterKeys } from '@librechat/data-schemas';
import {
+ AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT,
hasActivePiiFields,
hasActivePiiPatterns,
hasToolCallErrorPrefix,
@@ -271,6 +272,8 @@ export interface ToolExecuteOptions {
runFiles?: Pick<RunFileSession, 'isActive' | 'prepareTools' | 'withCodeExecution'>;
/** Trusted deployment gate for cooperative ordinary-tool cancellation. */
ordinaryToolCancellation?: boolean;
+ /** Deployment-configured cap for terminal output copied into durable receipts. */
+ backgroundCompletionResultMaxChars?: number;
/** Callback to process tool artifacts (code output files, file citations, etc.) */
toolEndCallback?: ToolEndCallback;
/** Durable internal-completion adapter, present only for an Event Actor invocation. */
@@ -5317,6 +5320,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
subagentTasks,
runFiles,
ordinaryToolCancellation = false,
+ backgroundCompletionResultMaxChars = AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT,
provisionFiles,
} = options;
@@ -5774,6 +5778,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
...(params.status === 'cancelled' ? { cancelled: true } : {}),
settledAt: new Date(current?.updatedAt ?? Date.now()),
...(completionPreregistered ? { completionWakeup: true } : {}),
+ ...(completionAdmission?.persistResult != null
+ ? { completionReceipt: true }
+ : {}),
...(current?.resultClaim != null
? {
resultClaim: {
@@ -5794,10 +5801,40 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
task.id,
);
const backgroundTask = resolveBackgroundTask();
+ let durableReceiptReady = false;
+ let durableReceiptAmbiguous = false;
+ const persistDurableReceipt = async (receipt: {
+ status: 'completed' | 'error' | 'cancelled';
+ output?: string;
+ }): Promise<boolean> => {
+ if (completionAdmission?.persistResult == null) {
+ return false;
+ }
+ try {
+ return await completionAdmission.persistResult({
+ status: receipt.status,
+ output: truncateMiddle(
+ receipt.output ?? '',
+ backgroundCompletionResultMaxChars,
+ ),
+ settledAt: backgroundTask.settledAt,
+ });
+ } catch (receiptError) {
+ durableReceiptAmbiguous = true;
+ logger.warn(
+ `[background] Failed to persist independent result receipt for task ${task.id}:`,
+ receiptError,
+ );
+ return false;
+ }
+ };
const retireFailedPersistence = async (
reason: string,
certainty: 'definite' | 'ambiguous',
): Promise<void> => {
+ if (durableReceiptAmbiguous) {
+ return;
+ }
if (completionAdmission == null) {
backgroundTaskRegistry.markCompletionPersistenceFailed(
backgroundUserId,
@@ -5852,6 +5889,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
};
if (!harvestEnabled || !persistBackgroundCodeResult) {
+ durableReceiptReady = await persistDurableReceipt({
+ status: params.status,
+ output: params.output ?? localTask?.result,
+ });
if (
backgroundToolCompletion == null ||
detachedReservation?.status === 'reserved'
@@ -5870,17 +5911,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
backgroundTask,
resolveBackgroundTask,
});
- if (!deliveryReady) {
+ if (!deliveryReady && !durableReceiptReady) {
await retireFailedPersistence(
'background tool result was not persisted',
'definite',
);
}
} catch (persistError) {
- await retireFailedPersistence(
- 'background tool result persistence failed',
- isContentFilterError(persistError) ? 'definite' : 'ambiguous',
- );
+ if (!durableReceiptReady) {
+ await retireFailedPersistence(
+ 'background tool result persistence failed',
+ isContentFilterError(persistError) ? 'definite' : 'ambiguous',
+ );
+ }
logger.warn(
`[background] Failed to persist result for task ${task.id}:`,
persistError,
@@ -5922,7 +5965,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
task.id,
params.artifact,
);
- if (completionPreregistered) {
+ if (completionPreregistered && !durableReceiptReady) {
await retireFailedPersistence(
'background code result had no durable message anchor',
'definite',
@@ -5930,6 +5973,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
return;
}
+ if (persisted.deliveryReady !== false) {
+ durableReceiptReady = await persistDurableReceipt({
+ status: params.status,
+ output: params.output ?? localTask?.result,
+ });
+ }
if (persisted.deliveryReady === false) {
await retireFailedPersistence(
'background code result was not persisted',
@@ -5943,26 +5992,38 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
persisted.attachments,
);
} catch (persistError) {
- if (completionPreregistered) {
- await retireFailedPersistence(
- 'background code result persistence failed',
- isContentFilterError(persistError) ? 'definite' : 'ambiguous',
- );
- }
if (isContentFilterError(persistError)) {
+ const blockedMessage =
+ persistError instanceof ContentFilterError
+ ? modelBoundContentFilterErrorMessage(persistError.body)
+ : persistError.body.message;
backgroundTaskRegistry.blockArtifact(
backgroundUserId,
backgroundConversationId,
task.id,
- persistError instanceof ContentFilterError
- ? modelBoundContentFilterErrorMessage(persistError.body)
- : persistError.body.message,
+ blockedMessage,
);
+ durableReceiptReady = await persistDurableReceipt({
+ status: 'error',
+ output: blockedMessage,
+ });
+ if (completionPreregistered && !durableReceiptReady) {
+ await retireFailedPersistence(
+ 'background code result was blocked by content policy',
+ 'definite',
+ );
+ }
logger.warn(
`[background] Generated code output for task ${task.id} was blocked by content policy.`,
);
return;
}
+ if (completionPreregistered && !durableReceiptReady) {
+ await retireFailedPersistence(
+ 'background code result persistence failed',
+ 'ambiguous',
+ );
+ }
logger.warn(
`[background] Failed to persist code result for task ${task.id}:`,
persistError,
diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts
index 55eb828bb2..84738597ad 100644
--- a/packages/api/src/agents/harvest.ts
+++ b/packages/api/src/agents/harvest.ts
@@ -40,6 +40,8 @@ export interface BackgroundToolResultState {
settledAt: Date;
/** This exact task owns a pre-registered automatic continuation delivery. */
completionWakeup?: true;
+ /** Independent receipts retain task-local delivery ownership. */
+ completionReceipt?: true;
resultClaim?: {
kind: 'manual' | 'wakeup';
claimId: string;
diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts
index 56090134f1..cfca8b0cfc 100644
--- a/packages/api/src/agents/index.ts
+++ b/packages/api/src/agents/index.ts
@@ -25,6 +25,7 @@ export * from './handlers';
export * from './guard';
export * from './harvest';
export * from './backgroundCompletion';
+export * from './backgroundClaims';
export * from './backgroundCompletionWakeup';
export * from './initialize';
export * from './legacy';
diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts
index 91f24aa8f5..e00a600654 100644
--- a/packages/api/src/agents/triggers/engine.spec.ts
+++ b/packages/api/src/agents/triggers/engine.spec.ts
@@ -589,6 +589,32 @@ describe('createAgentTriggerDeliveryEngine', () => {
expect(store.dead).toHaveBeenCalledWith(expect.objectContaining({ claimToken: 'claim-1' }));
});
+ it.each([false, true])(
+ 'offers configured receipt recovery when already exhausted is %s',
+ async (exhausted) => {
+ const store = storeWith({
+ claimNext: jest.fn(async () => delivery({ attempts: exhausted ? 1 : 0 })),
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ {
+ store,
+ now: () => START,
+ dispatch: async () => {
+ throw new Error('database unavailable');
+ },
+ },
+ { concurrency: 1, maxAttempts: 1, retryCapMs: 123_000 },
+ );
+ await engine.runTick();
+ expect(store.dead).toHaveBeenCalledWith(
+ expect.objectContaining({
+ receiptRetryAt: new Date(START.getTime() + 123_000),
+ error: expect.objectContaining({ retryable: true }),
+ }),
+ );
+ },
+ );
+
it('bounds a persisted last failure before exhausting its source', async () => {
const oversized = 'x'.repeat(3_000);
const store = storeWith({
diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts
index 164854a0e7..8fd6592a3b 100644
--- a/packages/api/src/agents/triggers/engine.ts
+++ b/packages/api/src/agents/triggers/engine.ts
@@ -183,6 +183,7 @@ export interface AgentTriggerDeliveryStore {
attempt: number;
error: AgentTriggerDeliveryFailure;
settledAt: Date;
+ receiptRetryAt?: Date;
}) => Promise<boolean>;
}
@@ -409,6 +410,9 @@ export function createAgentTriggerDeliveryEngine(
attempt: delivery.attempts,
error: recorded,
settledAt: now(),
+ ...(recorded.retryable && {
+ receiptRetryAt: new Date(now().getTime() + retryCapMs),
+ }),
});
if (deadLettered) {
logger.error('[agent-triggers] delivery dead-lettered after exhausting retries', {
@@ -532,6 +536,9 @@ export function createAgentTriggerDeliveryEngine(
attempt,
error: recorded,
settledAt: attemptedAt,
+ ...(recorded.retryable && {
+ receiptRetryAt: new Date(attemptedAt.getTime() + retryCapMs),
+ }),
});
if (deadLettered) {
logger.error('[agent-triggers] delivery dead-lettered', {
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index bd3977598b..f3b6b4c103 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -1,4 +1,5 @@
import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@@ -180,6 +181,7 @@ describe('durable agent trigger service', () => {
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
expect.objectContaining({
workerCapabilities: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
@@ -218,6 +220,7 @@ describe('durable agent trigger service', () => {
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
expect.objectContaining({
workerCapabilities: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
],
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index 029312caee..9775bf5661 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -1,4 +1,5 @@
import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@@ -112,6 +113,9 @@ export interface AgentTriggerDeliveryPersistence {
completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];
retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];
renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];
+ persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
+ getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
+ releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
@@ -168,6 +172,19 @@ export interface AgentTriggerService {
options?: { onlyIfUnclaimed?: boolean; onlyIfDead?: boolean },
) => Promise<boolean>;
renewProducerLease: (deliveryKey: string, sourceId: string, leaseUntil: Date) => Promise<boolean>;
+ persistBackgroundToolResult: (input: {
+ deliveryKey: string;
+ sourceId: string;
+ result: {
+ status: 'completed' | 'error' | 'cancelled';
+ output: string;
+ settledAt: Date;
+ };
+ }) => Promise<boolean>;
+ getBackgroundToolResultClaim: (
+ input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
+ ) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
+ releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
drainUser: (userId: string) => Promise<void>;
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
@@ -183,6 +200,7 @@ function createDeliveryStore(
methods.claimNextAgentTriggerDelivery({
...input,
workerCapabilities: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
...(supportsDetachedActionCompletion()
@@ -600,6 +618,21 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
leaseUntil,
}),
),
+ persistBackgroundToolResult: (input) =>
+ runAsSystem(async () => {
+ const persist = requireMethods().persistAgentBackgroundToolResult;
+ return persist == null ? false : persist(input);
+ }),
+ getBackgroundToolResultClaim: (input) =>
+ runAsSystem(async () => {
+ const getClaim = requireMethods().getAgentBackgroundToolResultClaim;
+ return getClaim == null ? null : getClaim(input);
+ }),
+ releaseBackgroundToolResultClaims: (input) =>
+ runAsSystem(async () => {
+ const release = requireMethods().releaseAgentBackgroundToolResultClaims;
+ return release == null ? false : release(input);
+ }),
drainUser,
prepareUserPurge: (userId, fenceStartedAt, tenantId) =>
runAsSystem(async () =>
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index b582ba9a6d..c20a24d8f3 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -577,6 +577,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: true,
+ completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
});
});
@@ -593,6 +594,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: false,
+ completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
});
});
@@ -609,9 +611,31 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: true,
+ completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: true,
});
});
+
+ it('accepts a bounded durable completion result limit', () => {
+ const result = configSchema.safeParse({
+ version: '1.0',
+ endpoints: { agents: { backgroundTasks: { completionResultMaxChars: 4096 } } },
+ });
+
+ expect(result.success).toBe(true);
+ if (result.success) {
+ expect(result.data.endpoints?.agents?.backgroundTasks?.completionResultMaxChars).toBe(4096);
+ }
+ });
+
+ it.each([0, 64 * 1024 + 1])('rejects an unsafe completion result limit: %s', (limit) => {
+ expect(
+ configSchema.safeParse({
+ version: '1.0',
+ endpoints: { agents: { backgroundTasks: { completionResultMaxChars: limit } } },
+ }).success,
+ ).toBe(false);
+ });
});
describe('speechTab schema', () => {
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 0b0ec4f21e..c39109a7a6 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -22,6 +22,9 @@ import {
MIN_BALANCE_RESERVATION_TTL_MS,
DEFAULT_BALANCE_RESERVATION_TTL_MS,
} from './balance';
+
+export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;
+export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
import {
MAX_SUBAGENTS,
MAX_SUBAGENTS_CEILING,
@@ -1465,6 +1468,15 @@ export const agentsEndpointSchema = baseEndpointSchema
backgroundTasks: z
.object({
completionWakeups: z.boolean().optional().default(true),
+ /** Maximum terminal output copied into the private completion receipt.
+ * Generated files remain governed by their separate attachment policy. */
+ completionResultMaxChars: z
+ .number()
+ .int()
+ .min(1)
+ .max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
+ .optional()
+ .default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
/** Cooperative cancellation for process-local ordinary tools. Off
* by default so existing deployments opt into the new control. */
ordinaryToolCancellation: z.boolean().optional().default(false),
diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts
index 0b463796ad..4a2a7f5a23 100644
--- a/packages/data-schemas/src/index.ts
+++ b/packages/data-schemas/src/index.ts
@@ -67,6 +67,8 @@ export {
isCompactionSemanticIndexProjection,
} from './types/compaction';
export {
+ AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts
index 86c370d375..92ff017399 100644
--- a/packages/data-schemas/src/methods/conversation.spec.ts
+++ b/packages/data-schemas/src/methods/conversation.spec.ts
@@ -2230,10 +2230,14 @@ describe('Conversation Operations', () => {
const deleteAgentQueuedTurns = jest.fn(async () => {
transitions.push('queue-retired');
});
+ const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => {
+ transitions.push('trigger-results-erased');
+ });
const scopedMethods = createConversationMethods(mongoose, {
getMessages,
deleteMessages,
deleteAgentQueuedTurns,
+ eraseAgentTriggerDeliveryConversationResults,
});
await scopedMethods.deleteConvos(
@@ -2249,7 +2253,99 @@ describe('Conversation Operations', () => {
expect(deleteAgentQueuedTurns).toHaveBeenCalledWith('user123', [
{ conversationId, tenantId: 'tenant-1' },
]);
- expect(transitions).toEqual(['queue-retired', 'generation-drained']);
+ expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [
+ conversationId,
+ ]);
+ expect(transitions).toEqual([
+ 'queue-retired',
+ 'generation-drained',
+ 'trigger-results-erased',
+ ]);
+ });
+
+ it('preserves background receipts when a pre-delete drain fails', async () => {
+ const conversationId = uuidv4();
+ await Conversation.create({
+ conversationId,
+ user: 'user123',
+ endpoint: EModelEndpoint.agents,
+ });
+ const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined);
+ const scopedMethods = createConversationMethods(mongoose, {
+ getMessages,
+ deleteMessages,
+ eraseAgentTriggerDeliveryConversationResults,
+ });
+
+ await expect(
+ scopedMethods.deleteConvos(
+ 'user123',
+ { conversationId },
+ { beforeDelete: async () => Promise.reject(new Error('drain unavailable')) },
+ ),
+ ).rejects.toThrow('drain unavailable');
+
+ expect(await Conversation.findOne({ conversationId })).not.toBeNull();
+ expect(eraseAgentTriggerDeliveryConversationResults).not.toHaveBeenCalled();
+ });
+
+ it('finishes descendant bookkeeping and message cleanup when receipt erasure fails', async () => {
+ const parentId = uuidv4();
+ const childId = uuidv4();
+ const grandchildId = uuidv4();
+ const ids = [parentId, childId, grandchildId];
+ const project = await ChatProject.create({ user: 'user123', name: 'Receipt cleanup' });
+ await ConversationTag.create({ user: 'user123', tag: 'work', count: 3, position: 0 });
+ await Conversation.create(
+ ids.map((conversationId, depth) => ({
+ conversationId,
+ user: 'user123',
+ endpoint: EModelEndpoint.agents,
+ tags: ['work'],
+ chatProjectId: project._id.toString(),
+ ...(depth === 0
+ ? {}
+ : {
+ subagentThread: {
+ rootConversationId: parentId,
+ parentConversationId: ids[depth - 1],
+ parentMessageId: `message-${depth}`,
+ parentToolCallId: `call-${depth}`,
+ subagentType: 'agent-child',
+ subagentKind: 'agent',
+ depth,
+ },
+ }),
+ })),
+ );
+ const prepare = jest.fn(async () => undefined);
+ const erase = jest.fn(async () => {
+ throw new Error('receipt storage unavailable');
+ });
+ const scopedMethods = createConversationMethods(mongoose, {
+ getMessages,
+ deleteMessages,
+ prepareAgentTriggerConversationResultErasure: prepare,
+ eraseAgentTriggerDeliveryConversationResults: erase,
+ });
+
+ const result = await scopedMethods.deleteConvos('user123', { conversationId: parentId });
+ expect(result).toMatchObject({ deletedCount: 3, conversationIds: ids });
+ expect(prepare.mock.calls).toHaveLength(3);
+ expect(erase.mock.calls).toHaveLength(3);
+ expect(await Conversation.countDocuments({ user: 'user123' })).toBe(0);
+ expect((await ConversationTag.findOne({ user: 'user123', tag: 'work' }).lean())?.count).toBe(
+ 0,
+ );
+ expect((await ChatProject.findById(project._id).lean())?.conversationCount).toBe(0);
+ expect(deleteMessages).toHaveBeenCalledWith({
+ user: 'user123',
+ conversationId: { $in: ids },
+ });
+
+ await expect(
+ scopedMethods.deleteConvos('user123', { conversationId: parentId }, { allowEmpty: true }),
+ ).resolves.toMatchObject({ deletedCount: 0, conversationIds: [parentId] });
});
it('fails closed before deleting a conversation when queued-turn retirement fails', async () => {
@@ -2468,13 +2564,22 @@ describe('Conversation Operations', () => {
});
it('allows a single-conversation cleanup retry after topology and messages are gone', async () => {
- const result = await deleteConvos(
+ const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined);
+ const scopedMethods = createConversationMethods(mongoose, {
+ getMessages,
+ deleteMessages,
+ eraseAgentTriggerDeliveryConversationResults,
+ });
+ const result = await scopedMethods.deleteConvos(
'user123',
{ conversationId: 'already-absent' },
{ allowEmpty: true },
);
expect(result.deletedCount).toBe(0);
expect(result.conversationIds).toEqual(['already-absent']);
+ expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [
+ 'already-absent',
+ ]);
});
it('supports an idempotent empty recovery sweep without hiding storage failures', async () => {
diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts
index ec719499cb..3d44b0e641 100644
--- a/packages/data-schemas/src/methods/conversation.ts
+++ b/packages/data-schemas/src/methods/conversation.ts
@@ -512,6 +512,14 @@ export interface ConversationMethodDeps
user: string,
conversations: Array<{ conversationId: string; tenantId?: string; allTenants?: true }>,
) => Promise<void>;
+ eraseAgentTriggerDeliveryConversationResults?: (
+ user: string,
+ conversationIds: string[],
+ ) => Promise<void>;
+ prepareAgentTriggerConversationResultErasure?: (
+ user: string,
+ conversationIds: string[],
+ ) => Promise<void>;
}
export function createConversationMethods(
@@ -3235,7 +3243,18 @@ export function createConversationMethods(
})),
);
await options?.beforeDelete?.(waveIds);
+ await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds);
const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } });
+ if (result.deletedCount > 0) {
+ /** Result erasure is irreversible. Keep receipts intact when a
+ * pre-delete hook or the conversation delete itself fails, so a
+ * retained conversation cannot lose a receipt-only completion. */
+ try {
+ await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, waveIds);
+ } catch (error) {
+ logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error);
+ }
+ }
acknowledged &&= result.acknowledged;
deletedCount += result.deletedCount;
await reconcileDeletedWave(wave, result.deletedCount);
@@ -3266,6 +3285,11 @@ export function createConversationMethods(
allTenants: true,
})),
);
+ try {
+ await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, recoveryConversationIds);
+ } catch (error) {
+ logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error);
+ }
}
const deleteConvoResult: DeleteResult = { acknowledged, deletedCount };
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index f84fd5f1e2..3ddf1be4b4 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -325,6 +325,10 @@ export function createMethods(
getMessages: messageMethods.getMessages,
deleteMessages: messageMethods.deleteMessages,
searchMessages: messageMethods.searchMessages,
+ eraseAgentTriggerDeliveryConversationResults:
+ agentTriggerDeliveryMethods.eraseAgentTriggerDeliveryConversationResults,
+ prepareAgentTriggerConversationResultErasure:
+ agentTriggerDeliveryMethods.prepareAgentTriggerConversationResultErasure,
deleteAgentQueuedTurns: async (user, conversations) => {
/** Queued-turn ownership is ObjectId-backed. Conversation methods also
* support synthetic/non-ObjectId owners in embedded integrations and
diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts
index c8aae0d623..68b23bb1fc 100644
--- a/packages/data-schemas/src/methods/message.ts
+++ b/packages/data-schemas/src/methods/message.ts
@@ -726,6 +726,7 @@ export interface MessageMethods {
cancelled?: true;
settledAt: Date;
completionWakeup?: true;
+ completionReceipt?: true;
resultClaim?: {
kind: 'manual' | 'wakeup';
claimId: string;
@@ -1224,6 +1225,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
cancelled?: true;
settledAt: Date;
completionWakeup?: true;
+ completionReceipt?: true;
resultClaim?: {
kind: 'manual' | 'wakeup';
claimId: string;
@@ -1274,6 +1276,9 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
partPatch['content.$[part].tool_call.backgroundTask.cancelled'] =
backgroundTask.cancelled === true;
partPatch['content.$[part].tool_call.backgroundTask.settledAt'] = backgroundTask.settledAt;
+ if (backgroundTask.completionReceipt === true) {
+ partPatch['content.$[part].tool_call.backgroundTask.completionReceipt'] = true;
+ }
if (backgroundTask.completionWakeup === true) {
partPatch['content.$[part].tool_call.backgroundTask.completionWakeup'] = true;
} else {
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index 6ca5b42499..1f0fcc74f6 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -18,12 +18,14 @@ import {
type AgentTriggerDeliveryMethods,
} from './triggerDelivery';
import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
} from '~/types/triggerDelivery';
import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';
import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
+import { createConversationModel } from '../models/convo';
import { createUserModel } from '../models/user';
jest.mock('~/config/winston', () => ({
@@ -32,6 +34,7 @@ jest.mock('~/config/winston', () => ({
info: jest.fn(),
debug: jest.fn(),
}));
+jest.mock('~/models/plugins/mongoMeili', () => jest.fn());
const DB_SETUP_TIMEOUT_MS = 60_000;
const START = new Date('2026-08-17T12:00:00.000Z');
@@ -50,6 +53,7 @@ beforeAll(async () => {
LaneSequence = createAgentTriggerLaneSequenceModel(mongoose);
UserPurge = createAgentTriggerUserPurgeModel(mongoose);
User = createUserModel(mongoose);
+ createConversationModel(mongoose);
await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);
methods = createAgentTriggerDeliveryMethods(mongoose);
}, DB_SETUP_TIMEOUT_MS);
@@ -384,7 +388,7 @@ describe('agent trigger delivery methods', () => {
it('shields background completion work from workers that cannot resolve it', async () => {
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
const claimInput = {
@@ -400,7 +404,7 @@ describe('agent trigger delivery methods', () => {
...claimInput,
workerId: 'background-capable-worker',
claimToken: 'background-capable-claim',
- workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1],
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
}),
).resolves.toMatchObject({ id: queued.delivery.id });
});
@@ -412,7 +416,7 @@ describe('agent trigger delivery methods', () => {
enqueueInput({
deliveryKey: 'background-completion-producer-lease',
envelope: { event: { source } },
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: initialLease,
}),
);
@@ -449,6 +453,323 @@ describe('agent trigger delivery methods', () => {
).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil });
});
+ it('persists one private background result receipt independently of message rows', async () => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-result-receipt',
+ envelope: { event: { source } },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ const input = {
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ result: { status: 'completed' as const, output: 'durable output', settledAt: START },
+ };
+
+ await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true);
+ await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true);
+ await expect(
+ methods.getAgentBackgroundToolResult({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ }),
+ ).resolves.toEqual(input.result);
+ await expect(
+ methods.persistAgentBackgroundToolResult({
+ ...input,
+ result: { ...input.result, output: 'conflicting output' },
+ }),
+ ).resolves.toBe(false);
+ const ordinaryRead = await Delivery.findById(queued.delivery.id).lean();
+ expect(ordinaryRead).not.toHaveProperty('backgroundToolResult');
+ });
+
+ it('erases private result receipts when their conversation is deleted', async () => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const user = new mongoose.Types.ObjectId();
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-result-to-erase',
+ user,
+ envelope: {
+ event: { source },
+ target: { conversationId: 'conversation-to-delete' },
+ },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ await methods.persistAgentBackgroundToolResult({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ result: { status: 'completed', output: 'private output', settledAt: START },
+ });
+
+ await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation-to-delete']);
+
+ await expect(
+ methods.getAgentBackgroundToolResult({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ }),
+ ).resolves.toBeNull();
+ await expect(Delivery.exists({ _id: queued.delivery.id })).resolves.not.toBeNull();
+ await expect(
+ methods.persistAgentBackgroundToolResult({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ result: { status: 'completed', output: 'late private output', settledAt: START },
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it('does not make legacy string-owned conversation cleanup depend on an ObjectId cast', async () => {
+ await expect(
+ methods.prepareAgentTriggerConversationResultErasure('legacy-user', ['conversation']),
+ ).resolves.toBeUndefined();
+ await expect(
+ methods.eraseAgentTriggerDeliveryConversationResults('legacy-user', ['conversation']),
+ ).resolves.toBeUndefined();
+ });
+
+ it('recovers receipt erasure after a descendant row was already deleted', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ user,
+ envelope: { event: { source }, target: { conversationId: 'deleted-child' } },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ const lookup = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id };
+ await methods.persistAgentBackgroundToolResult({
+ ...lookup,
+ result: { status: 'completed', output: 'private child output', settledAt: START },
+ });
+ await methods.prepareAgentTriggerConversationResultErasure(user, ['deleted-child']);
+ await mongoose.models.Conversation.collection.insertOne({
+ user: user.toString(),
+ conversationId: 'deleted-child',
+ });
+ await methods.recoverAgentTriggerUserPurges();
+ await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.not.toBeNull();
+ await mongoose.models.Conversation.deleteOne({ user, conversationId: 'deleted-child' });
+ /** The conversation deletion committed, but the follow-up receipt erasure did not. */
+ await methods.recoverAgentTriggerUserPurges();
+ await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.toBeNull();
+ await expect(
+ Delivery.exists({
+ _id: queued.delivery.id,
+ backgroundToolResultDeletionPendingAt: { $exists: true },
+ }),
+ ).resolves.toBeNull();
+ await expect(
+ methods.persistAgentBackgroundToolResult({
+ ...lookup,
+ result: { status: 'completed', output: 'late write', settledAt: START },
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it.each([
+ ['receipt', 'BACKGROUND_TOOL_PRODUCER_LOST'],
+ ['death', 'BACKGROUND_TOOL_PRODUCER_LOST'],
+ ['receipt', 'PARENT_STATE_UNAVAILABLE'],
+ ['death', 'PARENT_STATE_UNAVAILABLE'],
+ ])('atomically fences result publication when %s wins before %s', async (winner, code) => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ envelope: { event: { source } },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ const owner = { workerId: 'worker', claimToken: 'claim', now: START };
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ ...owner,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
+ });
+ expect(claimed).not.toBeNull();
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id: claimed!.id });
+ const write = () =>
+ methods.persistAgentBackgroundToolResult({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ result: { status: 'completed', output: 'late output', settledAt: START },
+ });
+ const dead = () =>
+ methods.deadLetterAgentTriggerDelivery({
+ ...owner,
+ id: claimed!.id,
+ attempt: attempt!,
+ settledAt: START,
+ error: transientFailure({ code, retryable: code !== 'BACKGROUND_TOOL_PRODUCER_LOST' }),
+ });
+ await expect(winner === 'receipt' ? write() : dead()).resolves.toBe(true);
+ await expect(winner === 'receipt' ? dead() : write()).resolves.toBe(false);
+ });
+
+ it.each(['shield', 'legacy'] as const)(
+ 'recovers a committed receipt after generic retry exhaustion through the %s fence',
+ async (format) => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ envelope: { event: { source } },
+ requiredWorkerCapability:
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ const owner = { workerId: 'worker', claimToken: 'claim', now: START };
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ ...owner,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
+ });
+ const id = claimed!.id;
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id });
+ if (format !== 'shield') {
+ await Delivery.updateOne(
+ { _id: id },
+ {
+ $set: {
+ status: 'capability_leased',
+ leaseBy: owner.workerId,
+ claimToken: owner.claimToken,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ },
+ $unset: {
+ capabilityStatus: 1,
+ capabilityLeaseBy: 1,
+ capabilityLeaseUntil: 1,
+ capabilityClaimToken: 1,
+ },
+ },
+ );
+ }
+ const receipt = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id };
+ await methods.persistAgentBackgroundToolResult({
+ ...receipt,
+ result: { status: 'completed', output: 'saved output', settledAt: START },
+ });
+ const receiptRetryAt = new Date(START.getTime() + 300_000);
+ const terminal = {
+ ...owner,
+ id,
+ attempt: attempt!,
+ settledAt: START,
+ receiptRetryAt,
+ error: transientFailure({ code: 'PARENT_STATE_UNAVAILABLE' }),
+ };
+ await expect(
+ methods.deadLetterAgentTriggerDelivery({ ...terminal, claimToken: 'stale' }),
+ ).resolves.toBe(false);
+ expect((await Delivery.findById(id).lean())?.attempts).toBe(attempt);
+ await expect(methods.deadLetterAgentTriggerDelivery(terminal)).resolves.toBe(false);
+ expect(await Delivery.findById(id).lean()).toMatchObject({
+ attempts: 0,
+ claimAvailableAt: receiptRetryAt,
+ ...(format === 'shield'
+ ? { capabilityStatus: 'pending' }
+ : { status: 'capability_pending' }),
+ });
+ await expect(methods.getAgentBackgroundToolResult(receipt)).resolves.toMatchObject({
+ output: 'saved output',
+ });
+ const recovered = await methods.claimNextAgentTriggerDelivery({
+ ...owner,
+ claimToken: 'recovered',
+ now: receiptRetryAt,
+ leaseUntil: new Date(receiptRetryAt.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
+ });
+ expect(recovered?.id).toBe(id);
+ expect(
+ await methods.beginAgentTriggerDeliveryAttempt({
+ ...owner,
+ id,
+ claimToken: 'recovered',
+ now: receiptRetryAt,
+ }),
+ ).toBe(1);
+ await expect(
+ methods.deadLetterAgentTriggerDelivery({
+ ...terminal,
+ claimToken: 'recovered',
+ settledAt: receiptRetryAt,
+ error: transientFailure({ code: 'PARENT_NOT_FOUND', retryable: false }),
+ }),
+ ).resolves.toBe(true);
+ },
+ );
+
+ it('claims an independent result receipt before returning its output', async () => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const user = new mongoose.Types.ObjectId();
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-result-to-claim',
+ user,
+ envelope: {
+ event: {
+ source,
+ payload: { taskId: 'task-1', toolCallId: 'call-1', toolName: 'slow_tool' },
+ },
+ target: {
+ agentId: 'agent-1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ },
+ },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ await methods.persistAgentBackgroundToolResult({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ result: { status: 'completed', output: 'private output', settledAt: START },
+ });
+ const claim = {
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ userId: user.toString(),
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ agentId: 'agent-1',
+ claimId: 'delivery-claim-1',
+ limit: 1,
+ };
+
+ await expect(methods.claimAgentBackgroundToolResults(claim)).resolves.toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'task-1', output: 'private output' }],
+ });
+ await expect(
+ methods.claimAgentBackgroundToolResults({ ...claim, claimId: 'delivery-claim-2' }),
+ ).resolves.toEqual({ status: 'claimed', claimId: 'delivery-claim-1' });
+ await expect(
+ methods.getAgentBackgroundToolResultClaim({
+ sourceId: source.id,
+ userId: user.toString(),
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ taskId: 'task-1',
+ }),
+ ).resolves.toMatchObject({ kind: 'wakeup', claimId: 'delivery-claim-1' });
+ await expect(
+ methods.releaseAgentBackgroundToolResultClaims({
+ sourceId: source.id,
+ userId: user.toString(),
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ claimId: 'delivery-claim-1',
+ }),
+ ).resolves.toBe(true);
+ });
+
it('keeps capability-fenced work limited to capable workers through lease recovery', async () => {
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
index a4569db5fd..f2d12193b0 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -1,6 +1,7 @@
import { createHash } from 'node:crypto';
import type { FilterQuery, Model, Types } from 'mongoose';
import type {
+ AgentBackgroundToolResultReceipt,
AgentEventActorDetachedAction,
AgentTriggerDeliveryClaim,
AgentEventActorReceipt,
@@ -17,6 +18,8 @@ import type {
IAgentTriggerUserPurgeDocument,
} from '~/types/triggerDelivery';
import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@@ -129,6 +132,26 @@ export type AgentTriggerProducerLeaseStatus =
| { status: 'expired'; leaseUntil: Date }
| { status: 'missing' };
+export interface PersistAgentBackgroundToolResultInput {
+ deliveryKey: string;
+ sourceId: string;
+ result: AgentBackgroundToolResultReceipt;
+}
+
+export type AgentBackgroundToolResultClaim =
+ | { status: 'not_ready' }
+ | { status: 'claimed'; claimId: string }
+ | {
+ status: 'acquired';
+ results: Array<{
+ taskId: string;
+ toolCallId: string;
+ toolName: string;
+ status: AgentBackgroundToolResultReceipt['status'];
+ output: string;
+ }>;
+ };
+
export interface AgentTriggerDeliveryFence {
id: string;
workerId: string;
@@ -277,6 +300,37 @@ export interface AgentTriggerDeliveryMethods {
sourceId: string;
now: Date;
}) => Promise<AgentTriggerProducerLeaseStatus>;
+ persistAgentBackgroundToolResult: (
+ input: PersistAgentBackgroundToolResultInput,
+ ) => Promise<boolean>;
+ getAgentBackgroundToolResult: (input: {
+ deliveryKey: string;
+ sourceId: string;
+ }) => Promise<AgentBackgroundToolResultReceipt | null>;
+ getAgentBackgroundToolResultClaim: (input: {
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ taskId: string;
+ }) => Promise<AgentBackgroundToolResultReceipt['resultClaim'] | null>;
+ claimAgentBackgroundToolResults: (input: {
+ deliveryKey: string;
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ agentId?: string;
+ claimId: string;
+ limit?: number;
+ }) => Promise<AgentBackgroundToolResultClaim>;
+ releaseAgentBackgroundToolResultClaims: (input: {
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ claimId: string;
+ }) => Promise<boolean>;
settleAgentTriggerHandlingOutcome: (
input: SettleAgentTriggerHandlingOutcomeInput,
) => Promise<boolean>;
@@ -326,6 +380,7 @@ export interface AgentTriggerDeliveryMethods {
attempt: number;
error: AgentTriggerDeliveryFailure;
settledAt: Date;
+ receiptRetryAt?: Date;
},
) => Promise<boolean>;
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;
@@ -358,6 +413,14 @@ export interface AgentTriggerDeliveryMethods {
) => Promise<boolean>;
recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;
+ eraseAgentTriggerDeliveryConversationResults: (
+ user: string | Types.ObjectId,
+ conversationIds: string[],
+ ) => Promise<void>;
+ prepareAgentTriggerConversationResultErasure: (
+ user: string | Types.ObjectId,
+ conversationIds: string[],
+ ) => Promise<void>;
}
function normalizeFailure(error: AgentTriggerDeliveryFailure): AgentTriggerDeliveryFailure {
@@ -960,6 +1023,8 @@ export function createAgentTriggerDeliveryMethods(
if (
input.requiredWorkerCapability != null &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
+ input.requiredWorkerCapability !==
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1
) {
@@ -2122,7 +2187,12 @@ export function createAgentTriggerDeliveryMethods(
const renewed = await Delivery().updateOne(
{
deliveryKey: input.deliveryKey,
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ requiredWorkerCapability: {
+ $in: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ ],
+ },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
status: {
@@ -2161,7 +2231,12 @@ export function createAgentTriggerDeliveryMethods(
const delivery = await Delivery()
.findOne({
deliveryKey: input.deliveryKey,
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ requiredWorkerCapability: {
+ $in: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ ],
+ },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
})
@@ -2175,6 +2250,262 @@ export function createAgentTriggerDeliveryMethods(
: { status: 'expired', leaseUntil: delivery.producerLeaseUntil };
}
+ /** Stores terminal output on the pre-admitted delivery before attempting the
+ * parent-message projection. The first terminal receipt wins; exact retries
+ * are idempotent and conflicting rewrites fail closed. */
+ async function persistAgentBackgroundToolResult(
+ input: PersistAgentBackgroundToolResultInput,
+ ): Promise<boolean> {
+ const { result } = input;
+ if (
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 256 ||
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256 ||
+ !['completed', 'error', 'cancelled'].includes(result.status) ||
+ result.output.length > AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS ||
+ !(result.settledAt instanceof Date) ||
+ !Number.isFinite(result.settledAt.getTime())
+ ) {
+ throw new TypeError('Invalid background tool result receipt');
+ }
+ const identity = {
+ deliveryKey: input.deliveryKey,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ backgroundToolResultErasedAt: { $exists: false },
+ };
+ const updated = await Delivery().updateOne(
+ {
+ ...identity,
+ backgroundToolResult: { $exists: false },
+ status: { $nin: ['dead', 'capability_dead', 'succeeded'] },
+ capabilityStatus: { $ne: 'dead' },
+ },
+ { $set: { backgroundToolResult: result } },
+ { timestamps: false },
+ );
+ if (updated.matchedCount === 1) {
+ return true;
+ }
+ const existing = await Delivery()
+ .findOne(identity)
+ .select('+backgroundToolResult')
+ .lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
+ return (
+ existing?.backgroundToolResult?.status === result.status &&
+ existing.backgroundToolResult.output === result.output &&
+ existing.backgroundToolResult.settledAt.getTime() === result.settledAt.getTime()
+ );
+ }
+
+ async function getAgentBackgroundToolResult(input: {
+ deliveryKey: string;
+ sourceId: string;
+ }): Promise<AgentBackgroundToolResultReceipt | null> {
+ if (
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 256 ||
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256
+ ) {
+ throw new TypeError('Invalid background tool result receipt lookup');
+ }
+ const delivery = await Delivery()
+ .findOne({
+ deliveryKey: input.deliveryKey,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ })
+ .select('+backgroundToolResult')
+ .lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
+ return delivery?.backgroundToolResult ?? null;
+ }
+
+ async function getAgentBackgroundToolResultClaim(input: {
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ taskId: string;
+ }): Promise<AgentBackgroundToolResultReceipt['resultClaim'] | null> {
+ const delivery = await Delivery()
+ .findOne({
+ ...backgroundResultIdentity(input),
+ 'envelope.event.payload.taskId': input.taskId,
+ 'backgroundToolResult.resultClaim': { $exists: true },
+ })
+ .select('+backgroundToolResult')
+ .lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
+ return delivery?.backgroundToolResult?.resultClaim ?? null;
+ }
+
+ function backgroundResultIdentity(input: {
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ agentId?: string;
+ }) {
+ return {
+ user: input.userId,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ 'envelope.target.conversationId': input.conversationId,
+ 'envelope.target.parentMessageId': input.parentMessageId,
+ ...(input.agentId == null ? {} : { 'envelope.target.agentId': input.agentId }),
+ };
+ }
+
+ function projectBackgroundResult(delivery: {
+ envelope?: unknown;
+ backgroundToolResult?: AgentBackgroundToolResultReceipt;
+ }) {
+ const envelope = delivery.envelope;
+ if (envelope == null || typeof envelope !== 'object' || Array.isArray(envelope)) {
+ return null;
+ }
+ const event = (envelope as { event?: unknown }).event;
+ if (event == null || typeof event !== 'object' || Array.isArray(event)) {
+ return null;
+ }
+ const payload = (event as { payload?: unknown }).payload;
+ if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) {
+ return null;
+ }
+ const { taskId, toolCallId, toolName } = payload as Record<string, unknown>;
+ const receipt = delivery.backgroundToolResult;
+ if (
+ receipt == null ||
+ typeof taskId !== 'string' ||
+ typeof toolCallId !== 'string' ||
+ typeof toolName !== 'string'
+ ) {
+ return null;
+ }
+ return {
+ taskId,
+ toolCallId,
+ toolName,
+ status: receipt.status,
+ output: receipt.output,
+ };
+ }
+
+ /** Elects exactly one automatic continuation for an independently persisted
+ * result. A later stack layer may coalesce siblings after this ownership
+ * boundary, but the durable receipt itself is never dispatched unclaimed. */
+ async function claimAgentBackgroundToolResults(input: {
+ deliveryKey: string;
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ agentId?: string;
+ claimId: string;
+ limit?: number;
+ }): Promise<AgentBackgroundToolResultClaim> {
+ if (
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 256 ||
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256 ||
+ input.userId.length === 0 ||
+ input.conversationId.length === 0 ||
+ input.conversationId.length > 256 ||
+ input.parentMessageId.length === 0 ||
+ input.parentMessageId.length > 256 ||
+ input.claimId.length === 0 ||
+ input.claimId.length > 128
+ ) {
+ throw new TypeError('Invalid background tool result receipt claim');
+ }
+ const scope = backgroundResultIdentity(input);
+ const projection = 'envelope +backgroundToolResult';
+ let requested = await Delivery()
+ .findOne({ ...scope, deliveryKey: input.deliveryKey })
+ .select(projection)
+ .lean<Pick<IAgentTriggerDelivery, 'envelope' | 'backgroundToolResult'>>();
+ if (requested?.backgroundToolResult == null) {
+ return { status: 'not_ready' };
+ }
+ const existingClaim = requested.backgroundToolResult.resultClaim;
+ if (existingClaim != null && existingClaim.claimId !== input.claimId) {
+ return { status: 'claimed', claimId: existingClaim.claimId };
+ }
+ if (existingClaim == null) {
+ const claimed = await Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ backgroundToolResult: { $exists: true },
+ 'backgroundToolResult.resultClaim': { $exists: false },
+ },
+ {
+ $set: {
+ 'backgroundToolResult.resultClaim': {
+ kind: 'wakeup',
+ claimId: input.claimId,
+ claimedAt: new Date(),
+ },
+ },
+ },
+ { timestamps: false },
+ );
+ if (claimed.modifiedCount !== 1) {
+ requested = await Delivery()
+ .findOne({ ...scope, deliveryKey: input.deliveryKey })
+ .select(projection)
+ .lean<Pick<IAgentTriggerDelivery, 'envelope' | 'backgroundToolResult'>>();
+ const winner = requested?.backgroundToolResult?.resultClaim;
+ if (winner?.claimId !== input.claimId) {
+ return winner == null
+ ? { status: 'not_ready' }
+ : { status: 'claimed', claimId: winner.claimId };
+ }
+ }
+ }
+ const result = requested == null ? null : projectBackgroundResult(requested);
+ return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };
+ }
+
+ async function releaseAgentBackgroundToolResultClaims(input: {
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ claimId: string;
+ }): Promise<boolean> {
+ if (
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256 ||
+ input.userId.length === 0 ||
+ input.conversationId.length === 0 ||
+ input.conversationId.length > 256 ||
+ input.parentMessageId.length === 0 ||
+ input.parentMessageId.length > 256 ||
+ input.claimId.length === 0 ||
+ input.claimId.length > 128
+ ) {
+ throw new TypeError('Invalid background tool result receipt claim release');
+ }
+ const scope = backgroundResultIdentity(input);
+ await Delivery().updateMany(
+ { ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },
+ { $unset: { 'backgroundToolResult.resultClaim': 1 } },
+ { timestamps: false },
+ );
+ return (
+ (await Delivery().exists({
+ ...scope,
+ 'backgroundToolResult.resultClaim.claimId': input.claimId,
+ })) == null
+ );
+ }
+
async function settleAgentTriggerHandlingOutcome(
input: SettleAgentTriggerHandlingOutcomeInput,
): Promise<boolean> {
@@ -3076,10 +3407,26 @@ export function createAgentTriggerDeliveryMethods(
error: AgentTriggerDeliveryFailure;
availableAt: Date;
},
+ ): Promise<boolean> {
+ return retryDelivery(input);
+ }
+
+ async function retryDelivery(
+ input: Parameters<AgentTriggerDeliveryMethods['retryAgentTriggerDelivery']>[0],
+ recoverReceipt = false,
): Promise<boolean> {
const error = normalizeFailure(input.error);
+ const receiptFence = recoverReceipt
+ ? {
+ requiredWorkerCapability:
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ backgroundToolResult: { $exists: true },
+ }
+ : {};
+ const recoveredAttempt = recoverReceipt ? { attempts: 0 } : {};
const retryUpdate = (status: 'pending' | 'capability_pending') => ({
$set: {
+ ...recoveredAttempt,
status,
availableAt: input.availableAt,
claimAvailableAt: input.availableAt,
@@ -3108,9 +3455,10 @@ export function createAgentTriggerDeliveryMethods(
},
});
const shieldResult = await Delivery().updateOne(
- { _id: input.id, ...shieldCapabilityFence(input) },
+ { _id: input.id, ...shieldCapabilityFence(input), ...receiptFence },
{
$set: {
+ ...recoveredAttempt,
status: 'leased',
availableAt: input.availableAt,
capabilityStatus: 'pending',
@@ -3134,14 +3482,14 @@ export function createAgentTriggerDeliveryMethods(
return true;
}
const capabilityResult = await Delivery().updateOne(
- { _id: input.id, ...legacyCapabilityFence(input) },
+ { _id: input.id, ...legacyCapabilityFence(input), ...receiptFence },
retryUpdate('capability_pending'),
);
if (capabilityResult.modifiedCount === 1) {
return true;
}
const result = await Delivery().updateOne(
- { _id: input.id, ...ordinaryFence(input) },
+ { _id: input.id, ...ordinaryFence(input), ...receiptFence },
retryUpdate('pending'),
);
return result.modifiedCount === 1;
@@ -3152,9 +3500,23 @@ export function createAgentTriggerDeliveryMethods(
attempt: number;
error: AgentTriggerDeliveryFailure;
settledAt: Date;
+ receiptRetryAt?: Date;
},
): Promise<boolean> {
const error = normalizeFailure(input.error);
+ if (
+ error.retryable &&
+ input.receiptRetryAt != null &&
+ (await retryDelivery({ ...input, availableAt: input.receiptRetryAt }, true))
+ ) {
+ return false;
+ }
+ /** Publication must contend with every retryable terminal transition,
+ * including a receipt committed between recovery and the dead-letter CAS. */
+ const receiptFence =
+ error.retryable || error.code === 'BACKGROUND_TOOL_PRODUCER_LOST'
+ ? { backgroundToolResult: { $exists: false } }
+ : {};
const deadUpdate = (status: 'dead' | 'capability_dead') => ({
$set: { status, settledAt: input.settledAt, lastError: error },
$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },
@@ -3174,7 +3536,7 @@ export function createAgentTriggerDeliveryMethods(
},
});
const shieldDead = await Delivery().updateOne(
- { _id: input.id, ...shieldCapabilityFence(input) },
+ { _id: input.id, ...shieldCapabilityFence(input), ...receiptFence },
{
$set: {
// The pre-capability runtime already treats `capability_dead` as a
@@ -3201,16 +3563,20 @@ export function createAgentTriggerDeliveryMethods(
return true;
}
const capabilityDead = await Delivery().updateOne(
- { _id: input.id, ...legacyCapabilityFence(input) },
+ { _id: input.id, ...legacyCapabilityFence(input), ...receiptFence },
deadUpdate('capability_dead'),
);
if (capabilityDead.modifiedCount === 1) {
return true;
}
const dead = await Delivery()
- .findOneAndUpdate({ _id: input.id, ...ordinaryFence(input) }, deadUpdate('dead'), {
- new: true,
- })
+ .findOneAndUpdate(
+ { _id: input.id, ...ordinaryFence(input), ...receiptFence },
+ deadUpdate('dead'),
+ {
+ new: true,
+ },
+ )
.select('_id orderingKey batchMemberIds batchMembersSettledAt requeueCount')
.lean<
Pick<
@@ -3512,6 +3878,31 @@ export function createAgentTriggerDeliveryMethods(
if (!Number.isSafeInteger(limit) || limit <= 0) {
throw new TypeError('Agent trigger purge recovery limit must be a positive integer');
}
+ const pending = await Delivery()
+ .find({
+ backgroundToolResultDeletionPendingAt: { $exists: true },
+ })
+ .select('user envelope')
+ .sort({ backgroundToolResultDeletionPendingAt: 1, _id: 1 })
+ .limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
+ .lean<IAgentTriggerDelivery[]>();
+ for (const row of pending) {
+ const conversationId = (row.envelope as { target?: { conversationId?: string } }).target
+ ?.conversationId;
+ if (conversationId == null) {
+ continue;
+ }
+ const exists = await mongoose.models.Conversation.exists({ user: row.user, conversationId });
+ if (exists == null) {
+ await eraseAgentTriggerDeliveryConversationResults(row.user, [conversationId]);
+ } else {
+ await Delivery().updateOne(
+ { _id: row._id, backgroundToolResultDeletionPendingAt: { $exists: true } },
+ { $currentDate: { backgroundToolResultDeletionPendingAt: true } },
+ { timestamps: false },
+ );
+ }
+ }
const markers = await UserPurge()
.find({})
.sort({ updatedAt: 1, _id: 1 })
@@ -3565,6 +3956,44 @@ export function createAgentTriggerDeliveryMethods(
await UserPurge().deleteOne({ _id: user });
}
+ async function eraseAgentTriggerDeliveryConversationResults(
+ user: string | Types.ObjectId,
+ conversationIds: string[],
+ ): Promise<void> {
+ if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) {
+ return;
+ }
+ const erasedAt = new Date();
+ await Delivery().updateMany(
+ {
+ user,
+ 'envelope.target.conversationId': { $in: conversationIds },
+ },
+ {
+ $set: { backgroundToolResultErasedAt: erasedAt },
+ $unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },
+ },
+ { timestamps: false },
+ );
+ }
+
+ async function prepareAgentTriggerConversationResultErasure(
+ user: string | Types.ObjectId,
+ conversationIds: string[],
+ ): Promise<void> {
+ if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) return;
+ await Delivery().updateMany(
+ {
+ user,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ 'envelope.target.conversationId': { $in: conversationIds },
+ backgroundToolResultErasedAt: { $exists: false },
+ },
+ { $set: { backgroundToolResultDeletionPendingAt: new Date() } },
+ { timestamps: false },
+ );
+ }
+
return {
ensureAgentTriggerDeliveryIndexes,
enqueueAgentTriggerDelivery,
@@ -3578,6 +4007,11 @@ export function createAgentTriggerDeliveryMethods(
retireAgentTriggerDelivery,
renewAgentTriggerDeliveryProducerLease,
getAgentTriggerDeliveryProducerLease,
+ persistAgentBackgroundToolResult,
+ getAgentBackgroundToolResult,
+ getAgentBackgroundToolResultClaim,
+ claimAgentBackgroundToolResults,
+ releaseAgentBackgroundToolResultClaims,
settleAgentTriggerHandlingOutcome,
admitAgentEventActorAction,
releaseAgentEventActorAction,
@@ -3606,5 +4040,7 @@ export function createAgentTriggerDeliveryMethods(
cancelAgentTriggerUserPurge,
recoverAgentTriggerUserPurges,
deleteAgentTriggerDeliveriesByUser,
+ eraseAgentTriggerDeliveryConversationResults,
+ prepareAgentTriggerConversationResultErasure,
};
}
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
index 34cb661587..1de097809d 100644
--- a/packages/data-schemas/src/schema/triggerDelivery.ts
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -1,5 +1,6 @@
import { Schema } from 'mongoose';
import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';
+import { AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS } from '~/types/triggerDelivery';
const failureSchema = new Schema(
{
@@ -147,6 +148,38 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
capabilityClaimToken: { type: String },
/** Private process-owner heartbeat; never projected to legacy consumers. */
producerLeaseUntil: { type: Date, select: false },
+ backgroundToolResult: {
+ type: new Schema(
+ {
+ status: {
+ type: String,
+ enum: ['completed', 'error', 'cancelled'],
+ required: true,
+ },
+ output: {
+ type: String,
+ required: true,
+ maxlength: AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
+ },
+ settledAt: { type: Date, required: true },
+ resultClaim: {
+ type: new Schema(
+ {
+ kind: { type: String, enum: ['wakeup'], required: true },
+ claimId: { type: String, required: true, maxlength: 128 },
+ claimedAt: { type: Date, required: true },
+ },
+ { _id: false },
+ ),
+ required: false,
+ },
+ },
+ { _id: false },
+ ),
+ select: false,
+ },
+ backgroundToolResultErasedAt: { type: Date, select: false },
+ backgroundToolResultDeletionPendingAt: { type: Date, select: false },
attempts: { type: Number, required: true, default: 0, min: 0 },
availableAt: { type: Date, required: true },
envelopeBytes: { type: Number, min: 0 },
@@ -224,6 +257,7 @@ triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });
triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });
+triggerDeliverySchema.index({ backgroundToolResultDeletionPendingAt: 1 }, { sparse: true });
// Only successful rows receive expiresAt. Dead letters remain available until
// an operator explicitly requeues or removes them.
triggerDeliverySchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });
diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts
index 7e91c341fc..be019d65a2 100644
--- a/packages/data-schemas/src/types/triggerDelivery.ts
+++ b/packages/data-schemas/src/types/triggerDelivery.ts
@@ -15,7 +15,10 @@ export type AgentTriggerDeliveryStatus =
export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
'background_tool_completion_v1';
+export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 =
+ 'background_tool_completion_receipt_v2';
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';
+export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;
export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';
export interface AgentTriggerHandlingState {
@@ -77,6 +80,20 @@ export interface AgentTriggerDeliveryFailure {
status?: number;
}
+/** Private terminal result produced before a background-completion delivery is
+ * resolved. It is independent of the parent message projection because a fast
+ * task can settle before that response row exists. */
+export interface AgentBackgroundToolResultReceipt {
+ status: 'completed' | 'error' | 'cancelled';
+ output: string;
+ settledAt: Date;
+ resultClaim?: {
+ kind: 'wakeup';
+ claimId: string;
+ claimedAt: Date;
+ };
+}
+
export interface AgentTriggerDeliveryHistoryEntry {
attempt: number;
outcome: AgentTriggerDeliveryOutcome;
@@ -108,6 +125,11 @@ export interface IAgentTriggerDelivery {
capabilityClaimToken?: string;
/** Durable liveness evidence for process-owned capability work. */
producerLeaseUntil?: Date;
+ /** Durable source of truth for a background completion. */
+ backgroundToolResult?: AgentBackgroundToolResultReceipt;
+ /** Conversation-deletion fence preventing a late producer from restoring private output. */
+ backgroundToolResultErasedAt?: Date;
+ backgroundToolResultDeletionPendingAt?: Date;
attempts: number;
availableAt: Date;
envelopeBytes?: number;