Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛎️ feat: Wake Agents on Background Tool Completion (#15350)

Danny Avila2ae6c8aea9fe
diff --git a/api/server/controllers/agents/__tests__/callbacks.spec.js b/api/server/controllers/agents/__tests__/callbacks.spec.js
index 4eeff29642..a4195dccd4 100644
--- a/api/server/controllers/agents/__tests__/callbacks.spec.js
+++ b/api/server/controllers/agents/__tests__/callbacks.spec.js
@@ -1123,6 +1123,7 @@ describe('tool input validation marker', () => {
expect(data.result.tool_call.inputValidationError).toBe(true);
expect(contentParts[0].tool_call.inputValidationError).toBe(true);
+ expect(contentParts[0].tool_call.stepId).toBe('step-1');
expect(toolInputValidationErrors.size).toBe(0);
});
@@ -1171,6 +1172,7 @@ describe('tool input validation marker', () => {
expect(data.result.tool_call).not.toHaveProperty('inputValidationError');
expect(contentParts[0].tool_call).not.toHaveProperty('inputValidationError');
+ expect(contentParts[0].tool_call.stepId).toBe('step-1');
});
});
diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js
index 6dbe27f1bb..864273c0a8 100644
--- a/api/server/controllers/agents/callbacks.js
+++ b/api/server/controllers/agents/callbacks.js
@@ -611,10 +611,14 @@ function getDefaultHandlers({
});
}
aggregateContent({ event, data });
- if (validationDetails != null) {
- const runStep = stepMap?.get(data?.result?.id);
- const toolCall = contentParts?.[runStep?.index]?.tool_call;
- if (toolCall != null) {
+ const stepId = data?.result?.id;
+ const runStep = stepMap?.get(stepId);
+ const toolCall = contentParts?.[runStep?.index]?.tool_call;
+ if (toolCall != null) {
+ if (typeof stepId === 'string') {
+ toolCall.stepId = stepId;
+ }
+ if (validationDetails != null) {
toolCall.inputValidationError = true;
}
}
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index bcddf032f8..0d1eea95af 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -4,6 +4,8 @@ const {
createAgentEventContinueResolver,
createSubagentCompletionWakeupResolver,
SUBAGENT_COMPLETION_SOURCE,
+ createBackgroundToolCompletionWakeupResolver,
+ BACKGROUND_TOOL_COMPLETION_SOURCE,
GenerationJobManager,
} = require('@librechat/api');
const methods = require('~/models');
@@ -12,6 +14,10 @@ const subagentCompletionAdapter = createSubagentCompletionWakeupResolver({
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
});
+const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
+});
const eventActorAdapter = createAgentEventContinueResolver({
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
@@ -23,7 +29,10 @@ const service = createAgentTriggerService({
supportsDetachedActionCompletion: () => GenerationJobManager.supportsDetachedAgentEventActions,
prepareContinue: createAgentContinuationResolver({
eventActor: eventActorAdapter,
- internalSources: new Map([[SUBAGENT_COMPLETION_SOURCE, subagentCompletionAdapter]]),
+ internalSources: new Map([
+ [SUBAGENT_COMPLETION_SOURCE, subagentCompletionAdapter],
+ [BACKGROUND_TOOL_COMPLETION_SOURCE, backgroundToolCompletionAdapter],
+ ]),
}),
});
@@ -36,6 +45,8 @@ module.exports = {
getAgentTriggerDeliveryStatus: service.getDeliveryStatus,
getAgentTriggerDeadLetters: service.getDeadLetters,
requeueAgentTrigger: service.requeue,
+ retireAgentTrigger: service.retire,
+ renewAgentTriggerProducerLease: service.renewProducerLease,
drainAgentTriggerDeliveriesForUser: service.drainUser,
prepareAgentTriggerUserPurge: service.prepareUserPurge,
cancelAgentTriggerUserPurge: service.cancelUserPurge,
diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.js
index 9aa6e88f3c..48491d402b 100644
--- a/api/server/services/Agents/triggers.spec.js
+++ b/api/server/services/Agents/triggers.spec.js
@@ -8,7 +8,9 @@ jest.mock('@librechat/api', () => ({
createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),
createAgentContinuationResolver: jest.fn(() => jest.fn()),
createAgentEventContinueResolver: jest.fn(() => jest.fn()),
+ createBackgroundToolCompletionWakeupResolver: jest.fn(() => jest.fn()),
createSubagentCompletionWakeupResolver: jest.fn(() => jest.fn()),
+ BACKGROUND_TOOL_COMPLETION_SOURCE: 'background-tool-completion',
SUBAGENT_COMPLETION_SOURCE: 'subagent-completion',
GenerationJobManager: mockGenerationJobManager,
}));
diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js
new file mode 100644
index 0000000000..6fc0996e62
--- /dev/null
+++ b/api/server/services/Endpoints/agents/backgroundCompletion.js
@@ -0,0 +1,39 @@
+const {
+ createBackgroundToolCompletionWakeupHandler,
+ createBackgroundToolDeadClaimRecovery,
+ createBackgroundToolResultHandler,
+} = require('@librechat/api');
+const {
+ enqueueAgentTrigger,
+ renewAgentTriggerProducerLease,
+ retireAgentTrigger,
+} = require('../../Agents/triggers');
+
+const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeupHandler(
+ enqueueAgentTrigger,
+ retireAgentTrigger,
+ renewAgentTriggerProducerLease,
+);
+
+function createBackgroundToolResultPersistence({ req, updateToolCallResult }) {
+ return createBackgroundToolResultHandler({ req, updateToolCallResult });
+}
+
+function createDeadBackgroundToolClaimRecovery(
+ releaseBackgroundToolResultClaims,
+ getGenerationJob,
+ fenceGenerationClaim,
+) {
+ return createBackgroundToolDeadClaimRecovery(
+ retireAgentTrigger,
+ releaseBackgroundToolResultClaims,
+ getGenerationJob,
+ fenceGenerationClaim,
+ );
+}
+
+module.exports = {
+ preregisterBackgroundToolCompletion,
+ createBackgroundToolResultPersistence,
+ createDeadBackgroundToolClaimRecovery,
+};
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index aabb3c7e32..fa4d6710f0 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -66,6 +66,11 @@ const { checkPermission, findAccessibleResources } = require('~/server/services/
const AgentClient = require('~/server/controllers/agents/client');
const { processAddedConvo } = require('./addedConvo');
const subagentThreadTaskStore = require('./subagentThreadStore');
+const {
+ preregisterBackgroundToolCompletion,
+ createBackgroundToolResultPersistence,
+ createDeadBackgroundToolClaimRecovery,
+} = require('./backgroundCompletion');
const { logViolation } = require('~/cache');
const db = require('~/models');
@@ -408,6 +413,25 @@ const initializeClient = async ({
req,
updateToolCallResult: db.updateToolCallResult,
}),
+ backgroundToolCompletion: {
+ ...(completionWakeupsEnabled ? { preregister: preregisterBackgroundToolCompletion } : {}),
+ persist: createBackgroundToolResultPersistence({
+ req,
+ updateToolCallResult: db.updateToolCallResult,
+ }),
+ claim: db.claimBackgroundToolResults,
+ recoverDeadClaim: createDeadBackgroundToolClaimRecovery(
+ db.releaseBackgroundToolResultClaims,
+ (conversationId) => GenerationJobManager.getJob(conversationId),
+ ({ userId, conversationId, claimId }) =>
+ GenerationJobManager.fenceGenerationClaimForRecovery(
+ userId,
+ claimId,
+ conversationId,
+ conversationId,
+ ),
+ ),
+ },
emitAttachment: createAttachmentEmitter({ res, streamId, jobCreatedAt }),
emitPtcProgress: createPtcProgressEmitter({ res, streamId, jobCreatedAt }),
onSkillResolved: (skill, { agentId }) => {
diff --git a/client/src/components/Chat/Messages/Content/ContentParts.tsx b/client/src/components/Chat/Messages/Content/ContentParts.tsx
index 4cc420a24c..75d6c5ecf1 100644
--- a/client/src/components/Chat/Messages/Content/ContentParts.tsx
+++ b/client/src/components/Chat/Messages/Content/ContentParts.tsx
@@ -51,6 +51,42 @@ const getPartAgentId = (part: TMessageContentParts): string | undefined =>
(part as { agentId?: string })?.agentId ??
(part?.[ContentTypes.TOOL_CALL] as { agentId?: string } | undefined)?.agentId;
+const getPartStepId = (part: TMessageContentParts): string | undefined =>
+ (part?.[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined)?.stepId;
+
+const getToolGroupAnchorIndex = (parts: PartWithIndex[]): number => {
+ for (const { part, idx } of parts) {
+ if (getToolCallId(part) || part?.type === ContentTypes.TOOL_CALL) {
+ return idx;
+ }
+ }
+ return parts[0]?.idx ?? -1;
+};
+
+type ToolCallStepOwner = { agentId?: string; stepId: string };
+
+const getSiblingStepIds = (
+ part: TMessageContentParts,
+ ownersByToolCallId: ReadonlyMap<string, readonly ToolCallStepOwner[]>,
+): ReadonlySet<string> | undefined => {
+ if (getPartStepId(part) != null) {
+ return;
+ }
+ const toolCallId = getToolCallId(part);
+ const owners = ownersByToolCallId.get(toolCallId);
+ if (toolCallId === '' || owners == null) {
+ return;
+ }
+ const agentId = getPartAgentId(part);
+ const siblingStepIds = new Set<string>();
+ for (const owner of owners) {
+ if (agentId == null || owner.agentId == null || owner.agentId === agentId) {
+ siblingStepIds.add(owner.stepId);
+ }
+ }
+ return siblingStepIds.size === 0 ? undefined : siblingStepIds;
+};
+
const getToolGroupId = (parts: PartWithIndex[], fallbackScope: number): string => {
const firstPart = parts[0];
if (!firstPart) {
@@ -64,7 +100,12 @@ const getToolGroupId = (parts: PartWithIndex[], fallbackScope: number): string =
for (const { part, idx } of parts) {
const toolCallId = getToolCallId(part);
if (toolCallId) {
- return `tool:${toolCallId}`;
+ const agentId = getPartAgentId(part);
+ /** `stepId` is stamped only when a tool finishes, so it cannot be part
+ * of the render identity without remounting a live group at completion.
+ * Agent + provider id are available from the first render; message-wide
+ * occurrence numbering below distinguishes later reuse in the same row. */
+ return agentId == null ? `tool:${toolCallId}` : `tool:${toolCallId}:${agentId}`;
}
if (firstToolKeyIdx === undefined && part?.type === ContentTypes.TOOL_CALL) {
firstToolKeyIdx = getPartKeyIndex(part, idx);
@@ -185,6 +226,10 @@ type ContentPartsProps = {
resumeAuthors?: ReadonlyMap<number, string | undefined>;
/** Message-wide tool-group expansion overrides retained across phase slices. */
toolGroupExpansionState?: Map<string, ToolCallGroupExpansionState>;
+ /** Message-wide occurrence number for each grouped tool block's first tool. */
+ toolGroupOccurrenceByIndex?: ReadonlyMap<number, number>;
+ /** Completed step owners used to keep older attachments off a live repeated call. */
+ toolCallStepOwnersById?: ReadonlyMap<string, readonly ToolCallStepOwner[]>;
};
/**
@@ -217,6 +262,8 @@ const ContentPartsBody = memo(function ContentPartsBody({
contentIndices,
resumeAuthors,
toolGroupExpansionState,
+ toolGroupOccurrenceByIndex,
+ toolCallStepOwnersById,
}: ContentPartsProps) {
const { inlineAttachments, workspaceChanges } = useMemo(
() =>
@@ -226,6 +273,26 @@ const ContentPartsBody = memo(function ContentPartsBody({
[attachments, workspaceAttachmentsPartitioned],
);
const attachmentMap = useMemo(() => mapAttachments(inlineAttachments), [inlineAttachments]);
+ const resolvedToolCallStepOwners = useMemo(() => {
+ if (toolCallStepOwnersById != null) {
+ return toolCallStepOwnersById;
+ }
+ const owners = new Map<string, ToolCallStepOwner[]>();
+ for (const part of content ?? []) {
+ if (part == null) {
+ continue;
+ }
+ const toolCallId = getToolCallId(part);
+ const stepId = getPartStepId(part);
+ if (toolCallId === '' || stepId == null) {
+ continue;
+ }
+ const entries = owners.get(toolCallId) ?? [];
+ entries.push({ stepId, agentId: getPartAgentId(part) });
+ owners.set(toolCallId, entries);
+ }
+ return owners;
+ }, [toolCallStepOwnersById, content]);
const effectiveIsSubmitting = isLatestMessage ? isSubmitting : false;
const localToolGroupExpansionRef = useRef(new Map<string, ToolCallGroupExpansionState>());
const expansionState = toolGroupExpansionState ?? localToolGroupExpansionRef.current;
@@ -400,6 +467,8 @@ const ContentPartsBody = memo(function ContentPartsBody({
partAttachments={filterAttachmentsForPart(
attachmentMap[getToolCallId(part)],
getPartAgentId(part),
+ getPartStepId(part),
+ getSiblingStepIds(part, resolvedToolCallStepOwners),
)}
/>
);
@@ -415,6 +484,7 @@ const ContentPartsBody = memo(function ContentPartsBody({
isLast,
isLatestMessage,
messageId,
+ resolvedToolCallStepOwners,
],
);
@@ -437,6 +507,8 @@ const ContentPartsBody = memo(function ContentPartsBody({
partAttachments={filterAttachmentsForPart(
attachmentMap[getToolCallId(part)],
getPartAgentId(part),
+ getPartStepId(part),
+ getSiblingStepIds(part, resolvedToolCallStepOwners),
)}
hideAttachments
onToolExpand={onToolExpand}
@@ -454,6 +526,7 @@ const ContentPartsBody = memo(function ContentPartsBody({
isLast,
isLatestMessage,
messageId,
+ resolvedToolCallStepOwners,
],
);
@@ -490,22 +563,54 @@ const ContentPartsBody = memo(function ContentPartsBody({
}, [absoluteIndexAt, content]);
const postSteerAuthors = resumeAuthors ?? detectedResumeAuthors;
- const groupedParts = useMemo(
- () =>
- groupSequentialToolCalls(sequentialParts).map((group) => {
- if (group.type === 'single') {
- return group;
- }
- const groupId = getToolGroupId(group.parts, fallbackScope);
- const groupAttachments = group.parts.flatMap(
- ({ part }) =>
- filterAttachmentsForPart(attachmentMap[getToolCallId(part)], getPartAgentId(part)) ??
- [],
- );
- return { ...group, groupId, groupAttachments };
- }),
- [sequentialParts, attachmentMap, fallbackScope],
- );
+ const resolvedToolGroupOccurrences = useMemo(() => {
+ if (toolGroupOccurrenceByIndex != null) {
+ return toolGroupOccurrenceByIndex;
+ }
+ const occurrences = new Map<number, number>();
+ const counts = new Map<string, number>();
+ for (const group of groupSequentialToolCalls(sequentialParts)) {
+ if (group.type === 'single') {
+ continue;
+ }
+ const baseGroupId = getToolGroupId(group.parts, fallbackScope);
+ const occurrence = (counts.get(baseGroupId) ?? 0) + 1;
+ counts.set(baseGroupId, occurrence);
+ occurrences.set(getToolGroupAnchorIndex(group.parts), occurrence);
+ }
+ return occurrences;
+ }, [toolGroupOccurrenceByIndex, sequentialParts, fallbackScope]);
+
+ const groupedParts = useMemo(() => {
+ return groupSequentialToolCalls(sequentialParts).map((group) => {
+ if (group.type === 'single') {
+ return group;
+ }
+ const baseGroupId = getToolGroupId(group.parts, fallbackScope);
+ const occurrence =
+ resolvedToolGroupOccurrences.get(getToolGroupAnchorIndex(group.parts)) ?? 1;
+ /** Legacy rows lack run-step identity. Their provider ids may repeat,
+ * so preserve the first group's historic stable key and distinguish
+ * later occurrences by sequence rather than a shifting content index. */
+ const groupId = occurrence === 1 ? baseGroupId : `${baseGroupId}:occurrence:${occurrence}`;
+ const groupAttachments = group.parts.flatMap(
+ ({ part }) =>
+ filterAttachmentsForPart(
+ attachmentMap[getToolCallId(part)],
+ getPartAgentId(part),
+ getPartStepId(part),
+ getSiblingStepIds(part, resolvedToolCallStepOwners),
+ ) ?? [],
+ );
+ return { ...group, groupId, groupAttachments };
+ });
+ }, [
+ sequentialParts,
+ attachmentMap,
+ fallbackScope,
+ resolvedToolGroupOccurrences,
+ resolvedToolCallStepOwners,
+ ]);
/** The re-attribution node for a part resuming after a steer block, shared
* by the sequential path and the parallel renderer's sequential stretches. */
@@ -599,6 +704,8 @@ const ContentPartsBody = memo(function ContentPartsBody({
contentIndices={segmentIndices}
resumeAuthors={postSteerAuthors}
toolGroupExpansionState={expansionState}
+ toolGroupOccurrenceByIndex={resolvedToolGroupOccurrences}
+ toolCallStepOwnersById={resolvedToolCallStepOwners}
/>
);
};
diff --git a/client/src/components/Chat/Messages/Content/__tests__/ContentParts.integration.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/ContentParts.integration.test.tsx
index bb53b13b3d..c0158e1198 100644
--- a/client/src/components/Chat/Messages/Content/__tests__/ContentParts.integration.test.tsx
+++ b/client/src/components/Chat/Messages/Content/__tests__/ContentParts.integration.test.tsx
@@ -58,6 +58,7 @@ jest.mock('lucide-react', () => ({
}));
jest.mock('@librechat/client', () => ({
+ useMediaQuery: () => false,
Button: ({
children,
variant: _variant,
@@ -127,7 +128,12 @@ jest.mock('~/utils', () => {
const MCP_DELIMITER = '_mcp_';
-const makeMcpToolCall = (id: string, hasOutput = true): TMessageContentParts =>
+const makeMcpToolCall = (
+ id: string,
+ hasOutput = true,
+ stepId?: string,
+ agentId?: string,
+): TMessageContentParts =>
({
type: ContentTypes.TOOL_CALL,
[ContentTypes.TOOL_CALL]: {
@@ -135,6 +141,8 @@ const makeMcpToolCall = (id: string, hasOutput = true): TMessageContentParts =>
name: `getTinyImage${MCP_DELIMITER}Everything`,
args: '{}',
output: hasOutput ? 'image_returned' : '',
+ ...(stepId == null ? {} : { stepId }),
+ ...(agentId == null ? {} : { agentId }),
},
}) as unknown as TMessageContentParts;
@@ -151,6 +159,17 @@ const makeMcpToolCallWithoutId = (name: string, hasOutput = true): TMessageConte
const makeTextPart = (text: string): TMessageContentParts =>
({ type: ContentTypes.TEXT, text }) as unknown as TMessageContentParts;
+const makePhasePart = (start: number, end: number, label: string): TMessageContentParts =>
+ ({
+ type: ContentTypes.ACTIVITY_LABEL,
+ [ContentTypes.ACTIVITY_LABEL]: label,
+ activity_label_type: 'phase',
+ activity_start_index: start,
+ activity_end_index: end,
+ activity_count: end - start,
+ pending: false,
+ }) as unknown as TMessageContentParts;
+
const imageAttachment = (toolCallId: string, name = 'tiny.png'): TAttachment =>
({
filename: name,
@@ -248,6 +267,27 @@ describe('ContentParts integration: MCP image hoist and grouping', () => {
expect(screen.queryByTestId('attachment-group')).not.toBeInTheDocument();
});
+ it('keeps an earlier step attachment off a live repeated provider call', () => {
+ const content = [
+ makeMcpToolCall('call_0', true, 'step-1', 'agent_a'),
+ makeMcpToolCall('call_1', true, 'step-1', 'agent_a'),
+ makeTextPart('between runs'),
+ makeMcpToolCall('call_0', false, undefined, 'agent_a'),
+ makeMcpToolCall('call_2', false, undefined, 'agent_a'),
+ ];
+ const previousAttachment = {
+ ...imageAttachment('call_0', 'previous.png'),
+ agentId: 'agent_a',
+ stepId: 'step-1',
+ } as unknown as TAttachment;
+
+ renderContentParts({ ...baseProps, content, attachments: [previousAttachment] });
+
+ const groups = screen.getAllByTestId('attachment-group');
+ expect(groups).toHaveLength(1);
+ expect(groups[0]).toHaveAttribute('data-count', '1');
+ });
+
it('keeps a manually expanded completed tool group open when its content index shifts', () => {
const content = [makeMcpToolCall('t1'), makeMcpToolCall('t2')];
const nextContent = [makeTextPart('streamed preface'), ...content];
@@ -276,9 +316,87 @@ describe('ContentParts integration: MCP image hoist and grouping', () => {
);
});
+ it('keeps repeated legacy provider-id groups independently expanded', () => {
+ const content = [
+ makeMcpToolCall('call_0'),
+ makeMcpToolCall('call_1'),
+ makeTextPart('between batches'),
+ makeMcpToolCall('call_0'),
+ makeMcpToolCall('call_2'),
+ ];
+ const nextContent = [makeTextPart('streamed preface'), ...content];
+ const { rerender } = render(
+ <RecoilRoot>
+ <ContentParts {...baseProps} content={content} />
+ </RecoilRoot>,
+ );
+
+ const toggles = screen.getAllByRole('button', { name: 'Used 2 tools' });
+ fireEvent.click(toggles[1]);
+ expect(toggles[0]).toHaveAttribute('aria-expanded', 'false');
+ expect(toggles[1]).toHaveAttribute('aria-expanded', 'true');
+
+ rerender(
+ <RecoilRoot>
+ <ContentParts {...baseProps} content={nextContent} />
+ </RecoilRoot>,
+ );
+ const shiftedToggles = screen.getAllByRole('button', { name: 'Used 2 tools' });
+ expect(shiftedToggles[0]).toHaveAttribute('aria-expanded', 'false');
+ expect(shiftedToggles[1]).toHaveAttribute('aria-expanded', 'true');
+ });
+
+ it('counts repeated legacy provider-id groups across activity phases', () => {
+ const content = [
+ makeMcpToolCall('call_0'),
+ makeMcpToolCall('call_1'),
+ makePhasePart(0, 2, 'First phase'),
+ makeMcpToolCall('call_0'),
+ makeMcpToolCall('call_2'),
+ makePhasePart(3, 5, 'Second phase'),
+ ];
+ const shiftedContent = [
+ makeTextPart('streamed preface'),
+ makeMcpToolCall('call_0'),
+ makeMcpToolCall('call_1'),
+ makePhasePart(1, 3, 'First phase'),
+ makeMcpToolCall('call_0'),
+ makeMcpToolCall('call_2'),
+ makePhasePart(4, 6, 'Second phase'),
+ ];
+ const { rerender } = render(
+ <RecoilRoot>
+ <ContentParts {...baseProps} content={content} />
+ </RecoilRoot>,
+ );
+
+ fireEvent.click(screen.getByRole('button', { name: 'First phase' }));
+ fireEvent.click(screen.getByRole('button', { name: 'Second phase' }));
+ const toggles = screen.getAllByRole('button', { name: 'Used 2 tools' });
+ fireEvent.click(toggles[1]);
+
+ rerender(
+ <RecoilRoot>
+ <ContentParts {...baseProps} content={shiftedContent} />
+ </RecoilRoot>,
+ );
+ for (const phase of ['First phase', 'Second phase']) {
+ const phaseToggle = screen.getByRole('button', { name: phase });
+ if (phaseToggle.getAttribute('aria-expanded') !== 'true') {
+ fireEvent.click(phaseToggle);
+ }
+ }
+ const shiftedToggles = screen.getAllByRole('button', { name: 'Used 2 tools' });
+ expect(shiftedToggles[0]).toHaveAttribute('aria-expanded', 'false');
+ expect(shiftedToggles[1]).toHaveAttribute('aria-expanded', 'true');
+ });
+
it('keeps a running tool group open when an individual tool is expanded before completion', () => {
const runningContent = [makeMcpToolCall('t1', false), makeMcpToolCall('t2', false)];
- const completedContent = [makeMcpToolCall('t1'), makeMcpToolCall('t2')];
+ const completedContent = [
+ makeMcpToolCall('t1', true, 'step-1'),
+ makeMcpToolCall('t2', true, 'step-1'),
+ ];
const { rerender } = render(
<RecoilRoot>
diff --git a/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts b/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts
index 310f9bd0d8..fa56252a24 100644
--- a/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts
+++ b/client/src/hooks/SSE/__tests__/useStepHandler.spec.ts
@@ -1835,6 +1835,7 @@ describe('useStepHandler', () => {
expect(toolCallContent?.tool_call?.output).toBe('Tool result output');
expect(toolCallContent?.tool_call?.progress).toBe(1);
expect(toolCallContent?.tool_call?.inputValidationError).toBe(true);
+ expect(toolCallContent?.tool_call?.stepId).toBe('step-tool-1');
});
it('signals skill authoring when a completed create_file call targets a skill path', () => {
diff --git a/client/src/hooks/SSE/useStepHandler.ts b/client/src/hooks/SSE/useStepHandler.ts
index d7b91f855d..8270b13677 100644
--- a/client/src/hooks/SSE/useStepHandler.ts
+++ b/client/src/hooks/SSE/useStepHandler.ts
@@ -646,6 +646,7 @@ export default function useStepHandler({
id,
name,
args,
+ stepId: getNonEmptyValue([contentPart.tool_call.stepId, existingToolCall?.stepId]),
type: ToolCallTypes.TOOL_CALL,
auth: contentPart.tool_call.auth,
expires_at: contentPart.tool_call.expires_at,
@@ -966,6 +967,7 @@ export default function useStepHandler({
name: toolCall.name ?? '',
args: toolCall.args,
id: toolCallId,
+ stepId: runStep.id,
},
};
@@ -1229,6 +1231,7 @@ export default function useStepHandler({
name: toolCallDelta.name ?? '',
args: toolCallDelta.args ?? '',
id: toolCallId,
+ stepId: runStepDelta.id,
},
};
@@ -1283,7 +1286,7 @@ export default function useStepHandler({
const contentPart: Agents.MessageContentComplex = {
type: ContentTypes.TOOL_CALL,
- tool_call: result.tool_call,
+ tool_call: { ...result.tool_call, stepId },
};
// Use server's index, offset by the retained edit prefix
diff --git a/client/src/utils/__tests__/map.test.ts b/client/src/utils/__tests__/map.test.ts
index 060e04b223..6dc54dc42b 100644
--- a/client/src/utils/__tests__/map.test.ts
+++ b/client/src/utils/__tests__/map.test.ts
@@ -18,6 +18,42 @@ describe('filterAttachmentsForPart', () => {
expect(filterAttachmentsForPart(attachments, undefined)).toHaveLength(2);
});
+ it('routes repeated same-agent provider ids by host run-step identity', () => {
+ const attachments = [
+ att({ agentId: 'agent_a', stepId: 'step-1' }),
+ att({ agentId: 'agent_a', stepId: 'step-2', file_id: 'f2' }),
+ att({ agentId: 'agent_a', file_id: 'legacy' }),
+ ];
+ const filtered = filterAttachmentsForPart(attachments, 'agent_a', 'step-2');
+ expect(filtered?.map((attachment) => (attachment as { file_id?: string }).file_id)).toEqual([
+ 'f2',
+ 'legacy',
+ ]);
+ });
+
+ it('still scopes by step when a legacy part has no agent identity', () => {
+ const attachments = [att({ stepId: 'step-1' }), att({ stepId: 'step-2', file_id: 'f2' })];
+ expect(filterAttachmentsForPart(attachments, undefined, 'step-1')).toHaveLength(1);
+ });
+
+ it('keeps earlier owned steps off a live repeated call', () => {
+ const attachments = [
+ att({ agentId: 'agent_a', stepId: 'step-1' }),
+ att({ agentId: 'agent_a', stepId: 'step-live', file_id: 'live' }),
+ att({ agentId: 'agent_a', file_id: 'legacy' }),
+ ];
+ const filtered = filterAttachmentsForPart(
+ attachments,
+ 'agent_a',
+ undefined,
+ new Set(['step-1']),
+ );
+ expect(filtered?.map((attachment) => (attachment as { file_id?: string }).file_id)).toEqual([
+ 'live',
+ 'legacy',
+ ]);
+ });
+
it('returns the same reference when nothing is filtered (render stability)', () => {
const attachments = [att({ agentId: 'agent_a' })];
expect(filterAttachmentsForPart(attachments, 'agent_a')).toBe(attachments);
diff --git a/client/src/utils/map.ts b/client/src/utils/map.ts
index c67ff8f47e..9f43d5a502 100644
--- a/client/src/utils/map.ts
+++ b/client/src/utils/map.ts
@@ -25,22 +25,34 @@ export function mapAttachments(attachments: Array<t.TAttachment | null | undefin
}
/**
- * Filters a part's mapped attachments to those owned by the part's agent:
- * provider tool-call ids repeat across agents in handoff responses (e.g.
- * `call_0`), so `toolCallId` alone can route one agent's harvested files to a
- * sibling agent's card. An attachment without `agentId` matches any part
- * (single-agent runs and legacy rows); a part without `agentId` accepts all.
+ * Filters a part's mapped attachments to its saved-agent and host run-step
+ * owner. Provider tool-call ids repeat across agents and turns, so
+ * `toolCallId` alone can route sibling output to the wrong card. Missing
+ * attachment ownership remains a wildcard for legacy rows; a live part with
+ * no step excludes step identities already owned by message siblings.
*/
export function filterAttachmentsForPart(
attachments: t.TAttachment[] | undefined,
partAgentId?: string,
+ partStepId?: string,
+ siblingStepIds?: ReadonlySet<string>,
): t.TAttachment[] | undefined {
- if (!attachments || partAgentId == null) {
+ if (
+ !attachments ||
+ (partAgentId == null &&
+ partStepId == null &&
+ (siblingStepIds == null || siblingStepIds.size === 0))
+ ) {
return attachments;
}
const filtered = attachments.filter((attachment) => {
- const agentId = (attachment as { agentId?: string }).agentId;
- return agentId == null || agentId === partAgentId;
+ const agentMatches =
+ partAgentId == null || attachment.agentId == null || attachment.agentId === partAgentId;
+ const stepMatches =
+ partStepId != null
+ ? attachment.stepId == null || attachment.stepId === partStepId
+ : attachment.stepId == null || siblingStepIds?.has(attachment.stepId) !== true;
+ return agentMatches && stepMatches;
});
return filtered.length === attachments.length ? attachments : filtered;
}
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js
index 26e31c63ec..6bdcc429ab 100644
--- a/e2e/setup/fake-model.js
+++ b/e2e/setup/fake-model.js
@@ -1629,6 +1629,24 @@ function backgroundCollectResponses(messages, toolNames) {
};
}
+/** Fresh host-owned run started when the detached result becomes durable. */
+function backgroundCompletionResponses(text) {
+ if (!text.includes('background tool task has finished') || !text.includes('durable result')) {
+ return null;
+ }
+ const status = text.match(/"status":"(\w+)"/)?.[1] ?? 'missing';
+ const echo = text.match(/E2E slow echo: (bg-[\w-]+)/)?.[1] ?? 'missing';
+ return {
+ responses: [''],
+ resolveInvocation: async (_messages, options, runManager) => {
+ const agentId = getAgentIdFromInvocationOptions(options, runManager) ?? 'missing';
+ return {
+ response: `E2E background notified status=${status} echo=${echo} agent=${agentId}`,
+ };
+ },
+ };
+}
+
function parseHandoffScript(text) {
const encodedScript = getMarkerValue(text, HANDOFF_MARKER);
if (!encodedScript) {
@@ -2216,6 +2234,11 @@ function buildHandoffResponses(graph, parsed) {
}
function resolveResponses({ graph, messages, text, toolNames }) {
+ const backgroundCompletion = backgroundCompletionResponses(text);
+ if (backgroundCompletion) {
+ return backgroundCompletion;
+ }
+
const subagentActivity = subagentActivityResponses(text);
if (subagentActivity) {
return subagentActivity;
diff --git a/e2e/specs/mock/background-tools.spec.ts b/e2e/specs/mock/background-tools.spec.ts
index 21c6f00b38..a1f186e024 100644
--- a/e2e/specs/mock/background-tools.spec.ts
+++ b/e2e/specs/mock/background-tools.spec.ts
@@ -40,7 +40,7 @@ async function waitForBackgroundTool(page: Page) {
}
test.describe('background tool calls', () => {
- test('dispatches a tool in the background and collects its result on a later turn', async ({
+ test('dispatches a tool in the background and resumes the agent when it finishes', async ({
page,
}) => {
test.setTimeout(120000);
@@ -92,21 +92,27 @@ test.describe('background tool calls', () => {
await expect(dispatchAck).toBeVisible({ timeout: 30000 });
await expect(page).toHaveURL(/\/c\/(?!new)/, { timeout: 15000 });
- /** The tool finishes ~1.5s after dispatch, past the end of turn 1 — the
- * detached promise survives its originating turn. Give it time to land
- * in the registry before the collect turn polls. */
- await page.waitForTimeout(2500);
-
- /** Turn 2: the fake model recovers the task id from turn-1 history, calls
- * `check_background_task` with it, and echoes what the poll returned —
- * status + the tool's output retrieved from the cross-turn registry. */
- const collect = await sendMessage(page, `E2E_BACKGROUND_COLLECT:${runToken}`);
- expect(collect.ok()).toBeTruthy();
- await expect(
- messagesView(page).getByText(
- new RegExp(`E2E background collected status=completed echo=bg-${runToken}`),
- ),
- ).toBeVisible({ timeout: 30000 });
+ /** The tool finishes after turn 1. The host durably claims the result and
+ * starts a fresh run on the same agent thread without another user turn.
+ * Triggered runs do not share the browser's original SSE connection, so
+ * reload while waiting for the persisted assistant response. */
+ const notification = new RegExp(
+ `E2E background notified status=completed echo=bg-${runToken} agent=${createdAgentId}`,
+ );
+ await expect
+ .poll(
+ async () => {
+ await page.reload();
+ /** `reload()` resolves before the asynchronously fetched message
+ * list is hydrated. Anchor each observation to the already-known
+ * dispatch response so a fast polling interval cannot repeatedly
+ * sample the transient empty conversation. */
+ await expect(dispatchAck).toBeVisible({ timeout: 5000 });
+ return messagesView(page).getByText(notification).count();
+ },
+ { timeout: 30000, intervals: [1500] },
+ )
+ .toBeGreaterThan(0);
} finally {
await cleanupAgent(page, createdAgentId);
}
diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts
index 22cb2c5519..6e6fc52fff 100644
--- a/packages/api/src/agents/background.spec.ts
+++ b/packages/api/src/agents/background.spec.ts
@@ -396,7 +396,7 @@ describe('registerBackgroundTaskTool', () => {
);
});
- it('advertises automatic delivery only for wakeup-enabled subagents', () => {
+ it('advertises automatic delivery for wakeup-enabled background work', () => {
const registry: LCToolRegistry = new Map();
const manual = registerBackgroundTaskTool({ toolRegistry: registry, toolDefinitions: [] });
const manualDescription = manual.toolDefinitions[0].description ?? '';
@@ -409,10 +409,10 @@ describe('registerBackgroundTaskTool', () => {
});
expect(automatic.toolDefinitions).toHaveLength(1);
expect(automatic.toolDefinitions[0].description).toContain(
- 'Detached subagent tasks use automatic completion delivery',
+ 'Background tools and detached subagents use automatic completion delivery',
);
expect(automatic.toolDefinitions[0].description).toContain(
- 'Ordinary background tool tasks require polling',
+ 'Ordinary tool execution remains process-local',
);
});
});
@@ -944,7 +944,7 @@ describe('BackgroundTaskRegistryClass', () => {
expect(JSON.stringify(task)).not.toContain('raw failure');
});
- it('exposes reaped (timed-out) tasks to the heal path when harvest was armed at dispatch', () => {
+ it('keeps abort-resistant tasks nonterminal instead of exposing false timeout evidence', () => {
jest.useFakeTimers();
try {
const created = backgroundTaskRegistry.create({
@@ -959,9 +959,8 @@ describe('BackgroundTaskRegistryClass', () => {
throw new Error('unexpected capacity');
}
- /** Past the running TTL the sweeper reaps the task to an error; the
- * dispatch-time harvest flag keeps it visible to marker/re-anchor
- * delivery so the original card doesn't stay on "running" forever. */
+ /** The invocation owner may have requested abort, but registry age alone
+ * cannot prove that an external side effect stopped. */
jest.advanceTimersByTime(31 * 60 * 1000);
const delivery = getBackgroundCodeDelivery({
userId: 'reap_user',
@@ -970,10 +969,9 @@ describe('BackgroundTaskRegistryClass', () => {
});
expect(delivery).toEqual(
expect.objectContaining({
- status: 'error',
+ status: 'running',
toolCallId: 'call_reaped',
messageId: 'dispatch-msg',
- error: 'Background task timed out',
}),
);
} finally {
@@ -1047,7 +1045,7 @@ describe('BackgroundTaskRegistryClass', () => {
expect(registry.claimArtifact('u1', 'c1', created.task.id)).toBeUndefined();
});
- it('reaps a stuck running task past the running TTL (frees the slot)', () => {
+ it('does not reap an abort-resistant running task by wall clock alone', () => {
const registry = new BackgroundTaskRegistryClass();
const created = registry.create({
userId: 'u1',
@@ -1058,11 +1056,11 @@ describe('BackgroundTaskRegistryClass', () => {
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
- // backdate creation past the 30-min running TTL, then trigger a sweep
+ // Backdate past the abort deadline; only invocation settlement is terminal proof.
created.task.createdAt = Date.now() - 31 * 60 * 1000;
registry.list('u1', 'c1');
- expect(created.task.status).toBe('error');
- expect(created.task.error).toBe('Background task timed out');
+ expect(created.task.status).toBe('running');
+ expect(created.task.error).toBeUndefined();
});
it('sweeps an expired completed task on direct get() (no indefinite retention)', () => {
@@ -1387,6 +1385,284 @@ describe('runCheckBackgroundTask (singleton)', () => {
);
});
+ it('returns a same-generation result through a local claim that persistence can preserve', async () => {
+ const created = backgroundTaskRegistry.create({
+ userId: 'claim_user',
+ conversationId: 'claim_convo',
+ toolCallId: 'call_claim',
+ toolName: 'search_mcp_docs',
+ messageId: 'response-claim',
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ backgroundTaskRegistry.complete('claim_user', 'claim_convo', created.task.id, {
+ content: 'CLAIMED RESULT',
+ });
+ const retire = jest.fn(async () => true);
+ backgroundTaskRegistry.markCompletionWakeup('claim_user', 'claim_convo', created.task.id, {
+ renew: jest.fn(async () => true),
+ retire,
+ });
+ const claimBackgroundToolResult = jest
+ .fn()
+ .mockResolvedValueOnce({ status: 'not_ready' })
+ .mockResolvedValueOnce({ status: 'not_ready' })
+ .mockResolvedValueOnce({ status: 'acquired', results: [] });
+ const request = {
+ userId: 'claim_user',
+ conversationId: 'claim_convo',
+ args: { background_task_id: created.task.id },
+ toolCallId: 'poll-call',
+ agentId: 'agent_parent_1',
+ runId: 'poll-run',
+ claimBackgroundToolResult,
+ };
+
+ const persisting = JSON.parse(await runCheckBackgroundTask(request));
+ expect(persisting).toMatchObject({ status: 'result_persisting' });
+ expect(JSON.stringify(persisting)).not.toContain('CLAIMED RESULT');
+ const replay = JSON.parse(await runCheckBackgroundTask(request));
+ expect(replay).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });
+ expect(
+ backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,
+ ).toMatchObject({ kind: 'manual' });
+ expect(retire).toHaveBeenCalledTimes(1);
+ expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
+ onlyIfUnclaimed: true,
+ });
+ expect(claimBackgroundToolResult).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ messageId: 'response-claim',
+ taskId: created.task.id,
+ kind: 'manual',
+ }),
+ );
+ });
+
+ it('delivers a mandatory live-artifact poll without waiting for its dispatch row', async () => {
+ const created = backgroundTaskRegistry.create({
+ userId: 'artifact_poll_user',
+ conversationId: 'artifact_poll_convo',
+ toolCallId: 'call_artifact_poll',
+ toolName: 'artifact_tool',
+ messageId: 'response-artifact-poll',
+ liveArtifactPollRequired: true,
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ backgroundTaskRegistry.complete('artifact_poll_user', 'artifact_poll_convo', created.task.id, {
+ content: 'CONTENT WITH LIVE ARTIFACT',
+ artifact: { type: 'test-artifact' },
+ });
+ const retire = jest.fn(async () => true);
+ backgroundTaskRegistry.markCompletionWakeup(
+ 'artifact_poll_user',
+ 'artifact_poll_convo',
+ created.task.id,
+ { renew: jest.fn(async () => true), retire },
+ );
+ const claimBackgroundToolResult = jest.fn(async () => ({ status: 'not_ready' as const }));
+
+ const request = {
+ userId: 'artifact_poll_user',
+ conversationId: 'artifact_poll_convo',
+ args: { background_task_id: created.task.id },
+ toolCallId: 'poll-live-artifact',
+ runId: 'dispatch-run',
+ claimBackgroundToolResult,
+ };
+ const result = JSON.parse(await runCheckBackgroundTask(request));
+ const replay = JSON.parse(await runCheckBackgroundTask(request));
+
+ expect(result).toMatchObject({ status: 'completed', result: 'CONTENT WITH LIVE ARTIFACT' });
+ expect(replay).toMatchObject({ status: 'completed', result: 'CONTENT WITH LIVE ARTIFACT' });
+ expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
+ onlyIfUnclaimed: true,
+ });
+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2);
+ expect(
+ backgroundTaskRegistry.get('artifact_poll_user', 'artifact_poll_convo', created.task.id)
+ ?.resultClaim,
+ ).toMatchObject({ kind: 'manual' });
+ });
+
+ it('does not expose a result already assigned to an automatic continuation', async () => {
+ const created = backgroundTaskRegistry.create({
+ userId: 'scheduled_user',
+ conversationId: 'scheduled_convo',
+ toolCallId: 'call_scheduled',
+ toolName: 'search_mcp_docs',
+ messageId: 'response-scheduled',
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ backgroundTaskRegistry.complete('scheduled_user', 'scheduled_convo', created.task.id, {
+ content: 'PRIVATE UNTIL CONTINUATION',
+ });
+ backgroundTaskRegistry.markCompletionWakeup(
+ 'scheduled_user',
+ 'scheduled_convo',
+ created.task.id,
+ );
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'scheduled_user',
+ conversationId: 'scheduled_convo',
+ args: { background_task_id: created.task.id },
+ claimBackgroundToolResult: async () => ({ status: 'claimed' }),
+ }),
+ );
+ expect(result).toMatchObject({ status: 'delivery_scheduled' });
+ expect(JSON.stringify(result)).not.toContain('PRIVATE UNTIL CONTINUATION');
+ });
+
+ it('recovers a result through its dead batch-owner claim', async () => {
+ const created = backgroundTaskRegistry.create({
+ userId: 'dead_claim_user',
+ conversationId: 'dead_claim_convo',
+ toolCallId: 'call_dead_claim',
+ toolName: 'search_mcp_docs',
+ messageId: 'response-dead-claim',
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ backgroundTaskRegistry.complete('dead_claim_user', 'dead_claim_convo', created.task.id, {
+ content: 'RECOVERED CLAIMED RESULT',
+ });
+ backgroundTaskRegistry.markCompletionWakeup(
+ 'dead_claim_user',
+ 'dead_claim_convo',
+ created.task.id,
+ );
+ const claimBackgroundToolResult = jest
+ .fn()
+ .mockResolvedValueOnce({
+ status: 'claimed',
+ claim: { kind: 'wakeup', claimId: 'sibling-batch-root' },
+ })
+ .mockResolvedValueOnce({ status: 'acquired' });
+ const recoverDeadBackgroundToolClaim = jest.fn(async () => true);
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'dead_claim_user',
+ conversationId: 'dead_claim_convo',
+ args: { background_task_id: created.task.id },
+ toolCallId: 'poll-dead-claim',
+ runId: 'poll-run',
+ claimBackgroundToolResult,
+ recoverDeadBackgroundToolClaim,
+ }),
+ );
+
+ expect(result).toMatchObject({ status: 'completed', result: 'RECOVERED CLAIMED RESULT' });
+ expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith({
+ userId: 'dead_claim_user',
+ conversationId: 'dead_claim_convo',
+ messageId: 'response-dead-claim',
+ claimId: 'sibling-batch-root',
+ });
+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2);
+ });
+
+ it('does not expose a local result after its automatic resolver owns the lease', async () => {
+ const created = backgroundTaskRegistry.create({
+ userId: 'retire_user',
+ conversationId: 'retire_convo',
+ toolCallId: 'call_retire',
+ toolName: 'search_mcp_docs',
+ messageId: 'response-retire',
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ backgroundTaskRegistry.complete('retire_user', 'retire_convo', created.task.id, {
+ content: 'DO NOT DUPLICATE',
+ });
+ const retire = jest.fn(async () => false);
+ backgroundTaskRegistry.markCompletionWakeup('retire_user', 'retire_convo', created.task.id, {
+ renew: jest.fn(async () => true),
+ retire,
+ });
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'retire_user',
+ conversationId: 'retire_convo',
+ args: { background_task_id: created.task.id },
+ toolCallId: 'poll-retire',
+ runId: 'poll-run',
+ claimBackgroundToolResult: async () => ({ status: 'not_ready' }),
+ }),
+ );
+
+ expect(result).toMatchObject({ status: 'result_persisting' });
+ expect(JSON.stringify(result)).not.toContain('DO NOT DUPLICATE');
+ expect(
+ backgroundTaskRegistry.get('retire_user', 'retire_convo', created.task.id)?.resultClaim,
+ ).toBeUndefined();
+ expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
+ onlyIfUnclaimed: true,
+ });
+ expect(retire).toHaveBeenCalledWith(
+ 'dead completion recovered by same-generation manual poll',
+ {
+ onlyIfDead: true,
+ },
+ );
+ });
+
+ it('recovers a dead automatic completion into process-local polling', async () => {
+ const created = backgroundTaskRegistry.create({
+ userId: 'dead_user',
+ conversationId: 'dead_convo',
+ toolCallId: 'call_dead',
+ toolName: 'search_mcp_docs',
+ messageId: 'response-dead',
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ backgroundTaskRegistry.complete('dead_user', 'dead_convo', created.task.id, {
+ content: 'RECOVERED RESULT',
+ });
+ const retire = jest.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true);
+ backgroundTaskRegistry.markCompletionWakeup('dead_user', 'dead_convo', created.task.id, {
+ renew: jest.fn(async () => true),
+ retire,
+ });
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'dead_user',
+ conversationId: 'dead_convo',
+ args: { background_task_id: created.task.id },
+ toolCallId: 'poll-dead',
+ runId: 'poll-run',
+ claimBackgroundToolResult: async () => ({ status: 'not_ready' }),
+ }),
+ );
+
+ expect(result).toMatchObject({ status: 'completed', result: 'RECOVERED RESULT' });
+ expect(retire).toHaveBeenNthCalledWith(1, 'completion claimed by same-generation manual poll', {
+ onlyIfUnclaimed: true,
+ });
+ expect(retire).toHaveBeenNthCalledWith(
+ 2,
+ 'dead completion recovered by same-generation manual poll',
+ { onlyIfDead: true },
+ );
+ expect(
+ backgroundTaskRegistry.get('dead_user', 'dead_convo', created.task.id)
+ ?.completionPersistenceFailed,
+ ).toBe(true);
+ });
+
it('preserves local task lists when cross-replica subagent discovery is unavailable', async () => {
const ordinary = backgroundTaskRegistry.create({
userId: 'partial-list-owner',
@@ -1837,6 +2113,18 @@ describe('buildBackgroundHandleContent', () => {
expect(parsed.status).toBe('running');
expect(parsed.message).toContain(CHECK_BACKGROUND_TASK_NAME);
});
+
+ it('requires polling when the tool can return a process-local live artifact', () => {
+ const parsed = JSON.parse(
+ buildBackgroundHandleContent(
+ { id: 'artifact-task', toolName: 'artifact_tool', status: 'running' },
+ { completionWakeup: true, liveArtifactPollRequired: true },
+ ),
+ );
+
+ expect(parsed.message).toContain('must call check_background_task');
+ expect(parsed.message).toContain('do not end the turn');
+ });
});
describe('toolOptionsSchema', () => {
diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts
index 13c15dc687..6b33da66ea 100644
--- a/packages/api/src/agents/background.ts
+++ b/packages/api/src/agents/background.ts
@@ -9,13 +9,13 @@
* a backgrounded call and the poll call are both synchronous from the graph's
* view.
*
- * Scope: cross-turn parallelism on a single Node process. The run's abort
- * signal does not reach the detached invoke (the graph forwards only
- * `configurable`/`metadata` to the tool-execute handler, never `signal`), so
- * the floating promise keeps running past turn completion and its result stays
- * in the in-process registry for a later turn to poll. Two boundaries: results
- * are lost on restart and are not shared across replicas (durable follow-up),
- * and ephemeral request-scoped MCP tools (runtime `{{LIBRECHAT_BODY_*}}`
+ * Scope: execution remains owned by one Node process, independently of the
+ * dispatch turn's abort signal. The host gives the detached invoke a separate
+ * deadline signal and accepts a timeout only after the invoke settles; an
+ * abort-resistant tool remains indeterminate and pollable. Terminal results
+ * can also be persisted onto the invoking response so another run or replica
+ * can consume them, but process death during execution does not recreate the
+ * live tool. Ephemeral request-scoped MCP tools (runtime `{{LIBRECHAT_BODY_*}}`
* placeholders) are never backgrounded — their connection is torn down at
* request end, so the executor runs them in the foreground instead. Detached
* subagents use the separate host task store; Redis-backed hosts may route
@@ -50,6 +50,11 @@ import type {
} from '@librechat/agents';
import type { AgentToolOptions } from 'librechat-data-provider';
import type { CapabilityToolNames } from './selection';
+import {
+ BACKGROUND_TASK_TIMEOUT_MS,
+ type BackgroundToolDeadClaimRecovery,
+ type BackgroundToolWakeupAdmission,
+} from './backgroundCompletion';
import {
resolveToolOption,
getSelectionNames,
@@ -341,7 +346,7 @@ Provide a background_task_id to poll one task; omit it to list every background
const CHECK_BACKGROUND_TASK_WAKEUP_DESCRIPTION = `Check, control, and retrieve tool or subagent tasks previously dispatched in the background (with run_in_background: true).
-Provide a background_task_id to inspect one task; omit it to list every background task in this thread. Ordinary background tool tasks require polling to retrieve their results. Detached subagent tasks use automatic completion delivery: continue independent work or end the turn instead of repeatedly polling an unchanged running task, and the host will resume you when one finishes. Use this tool for explicit status, steer, queue, interrupt, cancel, or cancel_message actions, or as a fallback if automatic delivery is unavailable. Live subagent controls route across API replicas but do not survive a restart of the process that owns the executor. A completed subagent thread may be continued later through the subagent tool's durable thread id.`;
+Provide a background_task_id to inspect one task; omit it to list every background task in this thread. Background tools and detached subagents use automatic completion delivery: continue independent work or end the turn instead of repeatedly polling an unchanged running task, and the host will resume you when one finishes. Use this tool for explicit status, steer, queue, interrupt, cancel, or cancel_message actions, or as a fallback if automatic delivery is unavailable. Ordinary tool execution remains process-local and does not survive restart; once its result is persisted, completion delivery may continue on another replica. Live subagent controls route across API replicas but do not survive a restart of the process that owns the executor. A completed subagent thread may be continued later through the subagent tool's durable thread id.`;
function checkBackgroundTaskDescription(subagentCompletionWakeups: boolean): string {
return subagentCompletionWakeups
@@ -590,6 +595,8 @@ export interface BackgroundTask {
id: string;
toolName: string;
toolCallId: string;
+ /** Stable run-step identity; provider tool-call ids may repeat within one response. */
+ stepId?: string;
/** The dispatch turn's response messageId, for post-hoc result anchoring. */
messageId?: string;
/** The dispatching agent, disambiguating repeated provider tool-call ids
@@ -626,6 +633,23 @@ export interface BackgroundTask {
artifactBlocked?: boolean;
/** Error message when status === 'error'. */
error?: string;
+ /** One consumer owns presentation of the terminal result. A manual claim is
+ * copied into the durable receipt when the dispatch row settles. */
+ resultClaim?: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ claimedAt: number;
+ };
+ /** The declared tool may return a process-local live artifact. A terminal
+ * same-generation poll may therefore deliver from the local claim after it
+ * retires the unclaimed wakeup, without waiting for the dispatch row to
+ * finalize and deadlocking that same generation. */
+ liveArtifactPollRequired?: boolean;
+ completionWakeup?: boolean;
+ /** Process-local cancellation handle for the preregistered durable delivery.
+ * A same-generation manual claim retires it before exposing the result. */
+ completionWakeupRetire?: BackgroundToolWakeupAdmission['retire'];
+ completionPersistenceFailed?: boolean;
createdAt: number;
updatedAt: number;
}
@@ -649,16 +673,39 @@ export interface BackgroundTaskCapacityPermit {
const COMPLETED_TASK_TTL_MS = 60 * 60 * 1000;
const IDLE_BUCKET_TTL_MS = 6 * 60 * 60 * 1000;
-/** Max wall-clock a task may stay `running` before being reaped as timed-out,
- * so a detached call that never settles (hung network / lost MCP connection)
- * can't hold a running slot and exhaust the per-conversation cap forever. */
-const RUNNING_TASK_TTL_MS = 30 * 60 * 1000;
const MAX_RUNNING_PER_BUCKET = 10;
const MAX_TASKS_PER_BUCKET = 200;
const MAX_RESULT_CHARS = 100_000;
const MAX_ARTIFACT_CHARS = 10_000_000;
const GLOBAL_SWEEP_INTERVAL_MS = 60 * 1000;
+/**
+ * Requests cancellation at the invocation deadline, but accepts terminal
+ * timeout evidence only when the invocation subsequently settles. Rejecting
+ * this wrapper while the underlying tool can still mutate externally would
+ * publish a false failure and make a duplicate side effect appear safe.
+ */
+export function withBackgroundTaskTimeout<T>(
+ invocation: Promise<T>,
+ requestAbort: () => void,
+ timeoutMs: number = BACKGROUND_TASK_TIMEOUT_MS,
+): Promise<T> {
+ return new Promise<T>((resolve, reject) => {
+ const timeout = setTimeout(requestAbort, timeoutMs);
+ timeout.unref?.();
+ invocation.then(
+ (result) => {
+ clearTimeout(timeout);
+ resolve(result);
+ },
+ (error) => {
+ clearTimeout(timeout);
+ reject(error);
+ },
+ );
+ });
+}
+
let lastDispatchStamp = 0;
/**
* Strictly-increasing dispatch stamp. `createdAt` orders writers in the
@@ -719,14 +766,6 @@ export class BackgroundTaskRegistryClass {
private sweepBucketTasks(bucket: TaskBucket, now: number): void {
for (const [taskId, task] of bucket.tasks) {
- if (task.status === 'running' && now - task.createdAt > RUNNING_TASK_TTL_MS) {
- /** Reap a stuck task: freeing the running slot (it no longer counts
- * toward the cap) and letting the completed-task TTL evict it. */
- task.status = 'error';
- task.error = 'Background task timed out';
- task.updatedAt = now;
- continue;
- }
if (task.status !== 'running' && now - task.updatedAt > COMPLETED_TASK_TTL_MS) {
bucket.tasks.delete(taskId);
}
@@ -852,14 +891,14 @@ export class BackgroundTaskRegistryClass {
userId: string;
conversationId: string;
toolCallId: string;
+ stepId?: string;
toolName: string;
messageId?: string;
runId?: string;
agentId?: string;
- /** Set at dispatch when a settle-time harvest WILL run, so tasks that
- * never settle (reaped as timed out) still take the marker/heal path
- * instead of leaving the original card on "running" forever. */
+ /** Set at dispatch when a settle-time harvest WILL run. */
harvestStarted?: boolean;
+ liveArtifactPollRequired?: boolean;
capacityPermit?: BackgroundTaskCapacityPermit;
}): { task: BackgroundTask; isNew: boolean } | { atCapacity: true } {
const now = Date.now();
@@ -919,9 +958,11 @@ export class BackgroundTaskRegistryClass {
id: params.taskId ?? randomUUID(),
toolName: params.toolName,
toolCallId: params.toolCallId,
+ stepId: params.stepId,
messageId: params.messageId,
agentId: params.agentId,
...(params.harvestStarted === true ? { harvestStarted: true, harvestPending: true } : {}),
+ ...(params.liveArtifactPollRequired === true ? { liveArtifactPollRequired: true } : {}),
status: 'running',
createdAt: nextDispatchStamp(now),
updatedAt: now,
@@ -950,16 +991,18 @@ export class BackgroundTaskRegistryClass {
conversationId: string,
taskId: string,
result: { content: unknown; artifact?: unknown; harvestStarted?: boolean },
- ): void {
+ ): string {
+ const storedContent = toStoredContent(result.content);
this.update(userId, conversationId, taskId, {
status: 'completed',
- result: toStoredContent(result.content),
+ result: storedContent,
artifact: toStoredArtifact(taskId, result.artifact),
...(result.harvestStarted === true ? { harvestStarted: true, harvestPending: true } : {}),
/** Marks that an artifact existed even after `claimArtifact` clears it,
* so re-polls keep the "produced an artifact" note. */
artifactDelivered: false,
});
+ return storedContent;
}
/**
@@ -1010,6 +1053,7 @@ export class BackgroundTaskRegistryClass {
| {
toolName: string;
toolCallId: string;
+ stepId?: string;
messageId?: string;
harvestStarted?: boolean;
artifact: unknown;
@@ -1033,6 +1077,7 @@ export class BackgroundTaskRegistryClass {
return {
toolName: task.toolName,
toolCallId: task.toolCallId,
+ stepId: task.stepId,
messageId: task.messageId,
harvestStarted: task.harvestStarted,
artifact,
@@ -1071,6 +1116,77 @@ export class BackgroundTaskRegistryClass {
});
}
+ markCompletionWakeup(
+ userId: string,
+ conversationId: string,
+ taskId: string,
+ admission?: BackgroundToolWakeupAdmission,
+ ): void {
+ this.update(userId, conversationId, taskId, {
+ completionWakeup: true,
+ ...(admission == null ? {} : { completionWakeupRetire: admission.retire }),
+ });
+ }
+
+ markCompletionPersistenceFailed(userId: string, conversationId: string, taskId: string): void {
+ this.update(userId, conversationId, taskId, {
+ completionPersistenceFailed: true,
+ completionWakeupRetire: undefined,
+ });
+ }
+
+ async retireCompletionWakeup(
+ userId: string,
+ conversationId: string,
+ taskId: string,
+ reason: string,
+ options?: { onlyIfUnclaimed?: boolean; onlyIfDead?: boolean },
+ ): Promise<boolean> {
+ const task = this.get(userId, conversationId, taskId);
+ if (task?.completionWakeupRetire == null) {
+ return false;
+ }
+ const retired = await task.completionWakeupRetire(reason, options);
+ if (retired) {
+ task.completionWakeupRetire = undefined;
+ task.updatedAt = Date.now();
+ }
+ return retired;
+ }
+
+ claimResult(
+ userId: string,
+ conversationId: string,
+ taskId: string,
+ claim: { kind: 'manual' | 'wakeup'; claimId: string },
+ ): 'acquired' | 'replay' | 'claimed' | 'not_ready' {
+ const task = this.get(userId, conversationId, taskId);
+ if (task == null || task.status === 'running') {
+ return 'not_ready';
+ }
+ if (task.resultClaim == null) {
+ task.resultClaim = { ...claim, claimedAt: Date.now() };
+ task.updatedAt = Date.now();
+ return 'acquired';
+ }
+ return task.resultClaim.kind === claim.kind && task.resultClaim.claimId === claim.claimId
+ ? 'replay'
+ : 'claimed';
+ }
+
+ releaseResultClaim(
+ userId: string,
+ conversationId: string,
+ taskId: string,
+ claim: { kind: 'manual' | 'wakeup'; claimId: string },
+ ): void {
+ const task = this.buckets.get(this.key(userId, conversationId))?.tasks.get(taskId);
+ if (task?.resultClaim?.kind === claim.kind && task.resultClaim.claimId === claim.claimId) {
+ task.resultClaim = undefined;
+ task.updatedAt = Date.now();
+ }
+ }
+
/** Permanently removes a policy-rejected artifact and exposes only the raw-free policy error. */
blockArtifact(userId: string, conversationId: string, taskId: string, error: string): void {
this.update(userId, conversationId, taskId, {
@@ -1139,12 +1255,21 @@ export const backgroundTaskRegistry = new BackgroundTaskRegistryClass();
/** Content for the synthetic ToolMessage returned when a call is backgrounded. */
export function buildBackgroundHandleContent(
task: Pick<BackgroundTask, 'id' | 'toolName' | 'status'>,
+ options: { completionWakeup?: boolean; liveArtifactPollRequired?: boolean } = {},
): string {
+ let message: string;
+ if (options.liveArtifactPollRequired === true) {
+ message = `Started "${task.toolName}" in the background. This tool can return a live artifact, so you must call ${CHECK_BACKGROUND_TASK_NAME} with background_task_id "${task.id}" until it completes; do not end the turn expecting artifact delivery from an automatic continuation. If the settled result is content-only, the host may still resume you automatically.`;
+ } else if (options.completionWakeup === true) {
+ message = `Started "${task.toolName}" in the background. Continue independent work or end the turn; the host will resume you when task "${task.id}" finishes. Use ${CHECK_BACKGROUND_TASK_NAME} only for an explicit status check or as a fallback.`;
+ } else {
+ message = `Started "${task.toolName}" in the background. Call ${CHECK_BACKGROUND_TASK_NAME} with background_task_id "${task.id}" to check progress and retrieve the result; it persists on this server, so you may poll it later in this turn or in a following turn. Do not assume it has finished until you have polled and seen status "completed".`;
+ }
return JSON.stringify({
background_task_id: task.id,
tool: task.toolName,
status: task.status,
- message: `Started "${task.toolName}" in the background. Call ${CHECK_BACKGROUND_TASK_NAME} with background_task_id "${task.id}" to check progress and retrieve the result; it persists on this server, so you may poll it later in this turn or in a following turn. Do not assume it has finished until you have polled and seen status "completed".`,
+ message,
});
}
@@ -1378,6 +1503,19 @@ export async function runCheckBackgroundTask(params: {
agentId?: string;
runId?: string;
subagentTasks?: SubagentTaskConfig;
+ claimBackgroundToolResult?: (params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskId: string;
+ agentId?: string;
+ kind: 'manual';
+ claimId: string;
+ }) => Promise<
+ | { status: 'acquired' | 'not_found' | 'not_ready' }
+ | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }
+ >;
+ recoverDeadBackgroundToolClaim?: BackgroundToolDeadClaimRecovery;
}): Promise<string> {
const { userId, conversationId } = params;
const args = coerceArgsObject(params.args) ?? {};
@@ -1402,6 +1540,182 @@ export async function runCheckBackgroundTask(params: {
message: 'Control actions are supported only for subagent tasks.',
});
}
+ if (task.status !== 'running') {
+ if (
+ task.completionWakeup === true &&
+ task.completionPersistenceFailed !== true &&
+ params.claimBackgroundToolResult != null &&
+ task.messageId != null
+ ) {
+ const durableClaimInput = {
+ userId,
+ conversationId,
+ messageId: task.messageId,
+ taskId,
+ agentId: task.agentId,
+ kind: 'manual' as const,
+ claimId: invocationId,
+ };
+ let durableClaim = await params.claimBackgroundToolResult(durableClaimInput);
+ if (durableClaim.status === 'claimed') {
+ let recovered = false;
+ let recoveryUnavailable = false;
+ if (
+ durableClaim.claim?.kind === 'wakeup' &&
+ params.recoverDeadBackgroundToolClaim != null
+ ) {
+ try {
+ recovered = await params.recoverDeadBackgroundToolClaim({
+ userId,
+ conversationId,
+ messageId: task.messageId,
+ claimId: durableClaim.claim.claimId,
+ });
+ } catch (error) {
+ recoveryUnavailable = true;
+ logger.warn(
+ `[background] Failed to reconcile claimed completion for manual poll ${taskId}:`,
+ error,
+ );
+ }
+ }
+ if (!recovered) {
+ return JSON.stringify({
+ status: recoveryUnavailable ? 'result_persisting' : 'delivery_scheduled',
+ background_task_id: taskId,
+ message: recoveryUnavailable
+ ? 'The automatic delivery recovery is temporarily unavailable. Retry this poll shortly.'
+ : 'This result is already assigned to an automatic continuation.',
+ });
+ }
+ durableClaim = await params.claimBackgroundToolResult(durableClaimInput);
+ if (durableClaim.status === 'claimed') {
+ return JSON.stringify({
+ status: 'delivery_scheduled',
+ background_task_id: taskId,
+ message: 'This result is already assigned to another continuation.',
+ });
+ }
+ if (durableClaim.status !== 'acquired') {
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message:
+ 'The task is finished and its result is being recovered. Retry this poll shortly.',
+ });
+ }
+ }
+ if (durableClaim.status === 'not_found' || durableClaim.status === 'not_ready') {
+ const localReplay =
+ task.resultClaim?.kind === 'manual' && task.resultClaim.claimId === invocationId;
+ let localClaimNeedsNoDurableConfirmation =
+ localReplay && task.liveArtifactPollRequired === true;
+ if (!localReplay) {
+ /** Retire the still-unclaimed delivery before creating local
+ * ownership. A live resolver lease wins. Once that resolver is
+ * irreversibly dead-lettered, a dead-only repair reopens the
+ * process-local poll fallback without stealing live work. */
+ let retired = false;
+ try {
+ retired = await backgroundTaskRegistry.retireCompletionWakeup(
+ userId,
+ conversationId,
+ taskId,
+ 'completion claimed by same-generation manual poll',
+ { onlyIfUnclaimed: true },
+ );
+ if (!retired) {
+ retired = await backgroundTaskRegistry.retireCompletionWakeup(
+ userId,
+ conversationId,
+ taskId,
+ 'dead completion recovered by same-generation manual poll',
+ { onlyIfDead: true },
+ );
+ if (retired) {
+ localClaimNeedsNoDurableConfirmation = true;
+ backgroundTaskRegistry.markCompletionPersistenceFailed(
+ userId,
+ conversationId,
+ taskId,
+ );
+ }
+ }
+ } catch (error) {
+ logger.warn(
+ `[background] Failed to retire automatic completion for manual claim ${taskId}:`,
+ error,
+ );
+ }
+ if (!retired) {
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message:
+ 'The task is finished and completion ownership is being settled. Retry this poll shortly.',
+ });
+ }
+ const localClaim = backgroundTaskRegistry.claimResult(
+ userId,
+ conversationId,
+ taskId,
+ { kind: 'manual', claimId: invocationId },
+ );
+ if (localClaim === 'claimed') {
+ return JSON.stringify({
+ status: 'delivery_scheduled',
+ background_task_id: taskId,
+ message: 'This result is already assigned to an automatic continuation.',
+ });
+ }
+ if (localClaim === 'not_ready') {
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message:
+ 'The task is finished and its result is being made durable. Retry this poll shortly.',
+ });
+ }
+ if (task.liveArtifactPollRequired === true) {
+ /** The poll is executing inside the still-unfinished dispatch
+ * generation, so waiting for the durable row would require
+ * that generation to end before it can obey its mandatory
+ * live-artifact poll. The retired unclaimed wakeup plus this
+ * local manual claim is authoritative for this owner process;
+ * the persistence retry re-reads and copies the claim after
+ * the generation finalizes. */
+ localClaimNeedsNoDurableConfirmation = true;
+ }
+ }
+ /** Ordinary polls do not expose the local result until the durable
+ * row has copied this manual claim. The owner-process live-artifact
+ * exception above cannot wait for its own generation to finalize;
+ * its persister re-reads the local claim after finalization. */
+ if (!localClaimNeedsNoDurableConfirmation) {
+ const reconciledClaim = await params.claimBackgroundToolResult(durableClaimInput);
+ if (reconciledClaim.status === 'claimed') {
+ backgroundTaskRegistry.releaseResultClaim(userId, conversationId, taskId, {
+ kind: 'manual',
+ claimId: invocationId,
+ });
+ return JSON.stringify({
+ status: 'delivery_scheduled',
+ background_task_id: taskId,
+ message: 'This result is already assigned to an automatic continuation.',
+ });
+ }
+ if (reconciledClaim.status !== 'acquired') {
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message:
+ 'The task is finished and its result is being made durable. Retry this poll shortly.',
+ });
+ }
+ }
+ }
+ }
+ }
return JSON.stringify(serializeTask(task, { includeResult: true }));
}
@@ -1547,6 +1861,7 @@ export function claimBackgroundArtifact(params: {
taskId: string;
toolName: string;
toolCallId: string;
+ stepId?: string;
messageId?: string;
harvestStarted?: boolean;
artifact: unknown;
@@ -1589,6 +1904,7 @@ export function getBackgroundCodeDelivery(params: {
status: BackgroundTaskStatus;
toolName: string;
toolCallId: string;
+ stepId?: string;
messageId?: string;
agentId?: string;
harvestStarted?: boolean;
@@ -1615,6 +1931,7 @@ export function getBackgroundCodeDelivery(params: {
status: task.status,
toolName: task.toolName,
toolCallId: task.toolCallId,
+ stepId: task.stepId,
messageId: task.messageId,
agentId: task.agentId,
harvestStarted: task.harvestStarted,
diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts
new file mode 100644
index 0000000000..a6bd7dc205
--- /dev/null
+++ b/packages/api/src/agents/backgroundCompletion.ts
@@ -0,0 +1,50 @@
+/** Deadline after which an invocation owner requests cancellation. */
+export const BACKGROUND_TASK_TIMEOUT_MS: number = 30 * 60 * 1000;
+/** Three missed heartbeats prove the process-local executor has been lost. */
+export const BACKGROUND_TOOL_PRODUCER_LEASE_MS: number = 30_000;
+export const BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS: number = 10_000;
+
+/** Host-owned identity recorded before ordinary background tool work begins. */
+export interface BackgroundToolWakeupRegistration {
+ taskId: string;
+ toolCallId: string;
+ toolName: string;
+ userId: string;
+ tenantId?: string;
+ conversationId: string;
+ parentMessageId: string;
+ parentAgentId?: string;
+ createdAt: number;
+}
+
+export interface BackgroundToolWakeupRetireOptions {
+ /** Retire only before a resolver owns the delivery. */
+ onlyIfUnclaimed?: boolean;
+ /** Reconcile only after the delivery is irreversibly dead-lettered. */
+ onlyIfDead?: boolean;
+}
+
+/** Process-local handle for the durable delivery admitted before launch. */
+export interface BackgroundToolWakeupAdmission {
+ /** Renews durable proof that the process-local executor still owns work. */
+ renew: () => Promise<boolean>;
+ /** Retires a delivery whose terminal result can no longer be made durable.
+ * Manual polling requires an atomic unclaimed-only transition: once a
+ * resolver owns a lease, its prepared continuation cannot be cancelled. */
+ retire: (reason: string, options?: BackgroundToolWakeupRetireOptions) => Promise<boolean>;
+}
+
+/** Durable ownership repair used by a manual poll after an automatic
+ * continuation delivery has irreversibly dead-lettered. `claimId` names the
+ * batch-root delivery that owns every claimed sibling, not necessarily the
+ * polled task's own pre-admitted delivery. */
+export interface BackgroundToolDeadClaimRecoveryInput {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ claimId: string;
+}
+
+export type BackgroundToolDeadClaimRecovery = (
+ input: BackgroundToolDeadClaimRecoveryInput,
+) => Promise<boolean>;
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
new file mode 100644
index 0000000000..cd79c4ac52
--- /dev/null
+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
@@ -0,0 +1,458 @@
+import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
+import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
+import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
+import {
+ BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS,
+ createBackgroundToolCompletionWakeupHandler,
+ createBackgroundToolCompletionWakeupResolver,
+ createBackgroundToolDeadClaimRecovery,
+} from './backgroundCompletionWakeup';
+import { parseAgentTriggerEnvelope } from './triggers/envelope';
+
+const NOW = Date.parse('2026-08-30T12:00:00Z');
+
+function registration(overrides = {}) {
+ return {
+ taskId: 'task-1',
+ toolCallId: 'call-1',
+ toolName: 'slow_tool',
+ userId: 'user-1',
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ parentAgentId: 'agent_parent_1',
+ createdAt: NOW - 10,
+ ...overrides,
+ };
+}
+
+function envelope(registrationOverrides = {}) {
+ let value: unknown;
+ const notify = createBackgroundToolCompletionWakeupHandler(
+ async (next) => {
+ value = next;
+ return { deliveryKey: 'delivery-key-1' };
+ },
+ async () => true,
+ async () => true,
+ );
+ return notify(registration(registrationOverrides)).then(() => {
+ const parsed = parseAgentTriggerEnvelope(value);
+ if (parsed.mode !== 'continue') {
+ throw new Error('Expected a continue envelope');
+ }
+ return parsed;
+ });
+}
+
+function resolverMethods() {
+ const releaseBackgroundToolResultClaims = jest.fn(async () => true);
+ return {
+ releaseBackgroundToolResultClaims,
+ methods: {
+ getConvo: jest.fn(async () => ({ tenantId: 'tenant-1' })),
+ getMessages: jest.fn(async () => [
+ {
+ messageId: 'response-1',
+ parentMessageId: 'user-1',
+ isCreatedByUser: false,
+ createdAt: new Date(NOW - 5),
+ },
+ {
+ messageId: 'response-2',
+ parentMessageId: 'response-1',
+ isCreatedByUser: false,
+ createdAt: new Date(NOW),
+ },
+ ]),
+ claimBackgroundToolResults: jest.fn(async () => ({
+ status: 'acquired',
+ results: [
+ {
+ taskId: 'task-1',
+ toolCallId: 'call-1',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'done',
+ },
+ {
+ taskId: 'task-2',
+ toolCallId: 'call-2',
+ toolName: 'other_tool',
+ status: 'error',
+ output: 'failed safely',
+ },
+ ],
+ })),
+ releaseBackgroundToolResultClaims,
+ getAgentTriggerDeliveryProducerLease: jest.fn(
+ async (): Promise<AgentTriggerProducerLeaseStatus> => ({
+ status: 'live',
+ leaseUntil: new Date(NOW + 30_000),
+ }),
+ ),
+ },
+ };
+}
+
+describe('background tool completion wakeups', () => {
+ beforeEach(() => {
+ jest.useFakeTimers().setSystemTime(NOW);
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ it('pre-registers the exact task on the invoking response branch', async () => {
+ const enqueue = jest.fn<
+ ReturnType<EnqueueBackgroundToolCompletion>,
+ Parameters<EnqueueBackgroundToolCompletion>
+ >(async () => ({ deliveryKey: 'delivery-key-1' }));
+ const retire = jest.fn(async () => true);
+ const renew = jest.fn(async () => true);
+ const notify = createBackgroundToolCompletionWakeupHandler(enqueue, retire, renew);
+
+ const admission = await notify(registration());
+ expect(admission).not.toBe(false);
+
+ const [value, options] = enqueue.mock.calls[0]!;
+ expect(parseAgentTriggerEnvelope(value)).toMatchObject({
+ deliveryId: 'task-1',
+ principal: { userId: 'user-1', tenantId: 'tenant-1' },
+ target: {
+ agentId: 'agent_parent_1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'response-1',
+ },
+ event: {
+ type: 'background-tool.completion',
+ source: { id: 'background-tool-completion', type: 'internal' },
+ payload: { taskId: 'task-1', toolCallId: 'call-1', toolName: 'slow_tool' },
+ },
+ });
+ expect(options).toEqual({
+ orderingKey: 'background-tool-completion:conversation-1:task-1',
+ availableAt: new Date(NOW + 250),
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ producerLeaseUntil: new Date(NOW + 30_000),
+ });
+ if (admission !== false) {
+ await expect(admission.renew()).resolves.toBe(true);
+ }
+ expect(renew).toHaveBeenCalledWith(
+ 'delivery-key-1',
+ 'background-tool-completion',
+ new Date(NOW + 30_000),
+ );
+ if (admission !== false) {
+ await expect(admission.retire('result unavailable')).resolves.toBe(true);
+ }
+ expect(retire).toHaveBeenCalledWith(
+ 'delivery-key-1',
+ 'background-tool-completion',
+ 'result unavailable',
+ );
+ if (admission !== false) {
+ await expect(
+ admission.retire('manual poll elected', { onlyIfUnclaimed: true }),
+ ).resolves.toBe(true);
+ }
+ expect(retire).toHaveBeenLastCalledWith(
+ 'delivery-key-1',
+ 'background-tool-completion',
+ 'manual poll elected',
+ { onlyIfUnclaimed: true },
+ );
+ if (admission !== false) {
+ await expect(admission.retire('dead delivery recovered', { onlyIfDead: true })).resolves.toBe(
+ true,
+ );
+ }
+ expect(retire).toHaveBeenLastCalledWith(
+ 'delivery-key-1',
+ 'background-tool-completion',
+ 'dead delivery recovered',
+ { onlyIfDead: true },
+ );
+ });
+
+ it('retires the batch-root delivery before releasing all of its sibling claims', async () => {
+ const retire = jest.fn(async () => true);
+ const release = jest.fn(async () => true);
+ const getGenerationJob = jest.fn(async () => ({ status: 'complete' }));
+ const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ getGenerationJob,
+ fenceGenerationClaim,
+ );
+
+ await expect(
+ recover({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-1',
+ claimId: 'batch-root-delivery',
+ }),
+ ).resolves.toBe(true);
+
+ expect(retire).toHaveBeenCalledWith(
+ 'batch-root-delivery',
+ 'background-tool-completion',
+ 'dead background completion batch recovered by manual poll',
+ { onlyIfDead: true },
+ );
+ expect(getGenerationJob).toHaveBeenCalledTimes(2);
+ expect(getGenerationJob).toHaveBeenCalledWith('conversation-1');
+ expect(fenceGenerationClaim).toHaveBeenCalledWith({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ claimId: 'batch-root-delivery',
+ });
+ expect(release).toHaveBeenCalledWith({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-1',
+ kind: 'wakeup',
+ claimId: 'batch-root-delivery',
+ });
+ });
+
+ it('does not recover a dead delivery while its admitted generation is still active', async () => {
+ const retire = jest.fn(async () => true);
+ const release = jest.fn(async () => true);
+ const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ async () => ({
+ status: 'running',
+ metadata: { idempotencyClientRequestId: 'batch-root-delivery' },
+ }),
+ fenceGenerationClaim,
+ );
+
+ await expect(
+ recover({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-1',
+ claimId: 'batch-root-delivery',
+ }),
+ ).resolves.toBe(false);
+ expect(retire).not.toHaveBeenCalled();
+ expect(release).not.toHaveBeenCalled();
+ expect(fenceGenerationClaim).not.toHaveBeenCalled();
+ });
+
+ it('rechecks a generation published while its dead delivery is being retired', async () => {
+ const retire = jest.fn(async () => true);
+ const release = jest.fn(async () => true);
+ const getGenerationJob = jest
+ .fn()
+ .mockResolvedValueOnce(undefined)
+ .mockResolvedValueOnce({
+ status: 'running',
+ metadata: { idempotencyClientRequestId: 'batch-root-delivery' },
+ })
+ .mockResolvedValue({ status: 'complete' });
+ const fenceGenerationClaim = jest
+ .fn()
+ .mockResolvedValueOnce('started' as const)
+ .mockResolvedValueOnce('fenced' as const);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ getGenerationJob,
+ fenceGenerationClaim,
+ );
+ const input = {
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-1',
+ claimId: 'batch-root-delivery',
+ };
+
+ await expect(recover(input)).resolves.toBe(false);
+ expect(retire).toHaveBeenCalledTimes(1);
+ expect(release).not.toHaveBeenCalled();
+
+ await expect(recover(input)).resolves.toBe(true);
+ expect(retire).toHaveBeenCalledTimes(2);
+ expect(release).toHaveBeenCalledTimes(1);
+ });
+
+ it('retries claim release through an idempotently retired recovery receipt', async () => {
+ const retire = jest.fn(async () => true);
+ const release = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('release receipt lost'))
+ .mockResolvedValueOnce(true);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ async () => ({
+ status: 'complete',
+ }),
+ async () => 'fenced',
+ );
+ const input = {
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-1',
+ claimId: 'batch-root-delivery',
+ };
+
+ await expect(recover(input)).rejects.toThrow('release receipt lost');
+ await expect(recover(input)).resolves.toBe(true);
+ expect(retire).toHaveBeenCalledTimes(2);
+ expect(release).toHaveBeenCalledTimes(2);
+ });
+
+ it("keeps unfinished sibling tasks out of each other's delivery lanes", async () => {
+ const enqueue = jest.fn<
+ ReturnType<EnqueueBackgroundToolCompletion>,
+ Parameters<EnqueueBackgroundToolCompletion>
+ >(async () => ({ deliveryKey: 'delivery-key' }));
+ const notify = createBackgroundToolCompletionWakeupHandler(
+ enqueue,
+ async () => true,
+ async () => true,
+ );
+
+ await notify(registration({ taskId: 'task-slow' }));
+ await notify(registration({ taskId: 'task-fast' }));
+
+ expect(enqueue.mock.calls.map(([, options]) => options?.orderingKey)).toEqual([
+ 'background-tool-completion:conversation-1:task-slow',
+ 'background-tool-completion:conversation-1:task-fast',
+ ]);
+ });
+
+ it('reports skipped registration for an ephemeral invoking agent', async () => {
+ const enqueue = jest.fn(async () => ({ deliveryKey: 'delivery-key-1' }));
+ const retire = jest.fn(async () => true);
+ const notify = createBackgroundToolCompletionWakeupHandler(enqueue, retire, async () => true);
+
+ await expect(notify(registration({ parentAgentId: 'ephemeral-agent' }))).resolves.toBe(false);
+ expect(enqueue).not.toHaveBeenCalled();
+ expect(retire).not.toHaveBeenCalled();
+ });
+
+ it('claims a bounded sibling batch and continues from the latest branch leaf', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
+
+ expect(prepared).toMatchObject({ status: 'ready', parentMessageId: 'response-2' });
+ expect(prepared?.status === 'ready' && prepared.input).toContain('task-2');
+ expect(methods.claimBackgroundToolResults).toHaveBeenCalledWith(
+ expect.objectContaining({
+ messageId: 'response-1',
+ taskId: 'task-1',
+ kind: 'wakeup',
+ claimId: 'delivery-1',
+ }),
+ );
+ });
+
+ it('shares one bounded input budget across a full sibling batch', async () => {
+ const { methods } = resolverMethods();
+ const results = Array.from({ length: 8 }, (_, index) => ({
+ taskId: `task-${index}`,
+ toolCallId: `call-${index}`,
+ toolName: 'large_tool',
+ status: 'completed' as const,
+ output: `${index}:` + 'large result '.repeat(10_000),
+ }));
+ methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'acquired', results });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-large' });
+
+ expect(prepared?.status).toBe('ready');
+ if (prepared?.status === 'ready') {
+ expect(prepared.input.length).toBeLessThanOrEqual(BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS);
+ for (const result of results) {
+ expect(prepared.input).toContain(result.taskId);
+ }
+ expect(prepared.input).toContain('[truncated:');
+ }
+ });
+
+ it('releases every claimed sibling when admission definitely fails', async () => {
+ const { methods, releaseBackgroundToolResultClaims } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
+
+ expect(prepared?.status).toBe('ready');
+ if (prepared?.status === 'ready') {
+ await prepared.releaseOnDefiniteFailure?.();
+ }
+ expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
+ expect.objectContaining({ taskIds: ['task-1', 'task-2'], claimId: 'delivery-1' }),
+ );
+ });
+
+ it('defers without claiming while the invoking generation is active', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => ({ status: 'running' }),
+ });
+
+ await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ {
+ code: 'PARENT_NOT_READY',
+ deferWithoutAttempt: true,
+ },
+ );
+ expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
+ });
+
+ it('does not manufacture terminal evidence from wall-clock age', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults.mockResolvedValue({ status: 'missing', results: [] });
+ const deliveryEnvelope = await envelope({ createdAt: NOW - 7 * 24 * 60 * 60_000 });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'BACKGROUND_TOOL_RESULT_NOT_READY', retryable: true },
+ );
+ });
+
+ it('terminally rejects a missing result after its process-local producer is lost', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults.mockResolvedValue({ status: 'missing', results: [] });
+ methods.getAgentTriggerDeliveryProducerLease.mockResolvedValue({
+ status: 'expired',
+ leaseUntil: new Date(NOW - 1),
+ });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+
+ await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ {
+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',
+ retryable: false,
+ },
+ );
+ });
+});
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts
new file mode 100644
index 0000000000..814f2d319d
--- /dev/null
+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts
@@ -0,0 +1,504 @@
+import { randomUUID } from 'node:crypto';
+import { isEphemeralAgentId } from 'librechat-data-provider';
+import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
+import type {
+ AgentTriggerProducerLeaseStatus,
+ BackgroundToolResultClaim,
+ ConversationMethods,
+ IMessage,
+ MessageMethods,
+} from '@librechat/data-schemas';
+import type {
+ BackgroundToolDeadClaimRecovery,
+ BackgroundToolWakeupAdmission,
+ BackgroundToolWakeupRegistration,
+ BackgroundToolWakeupRetireOptions,
+} from './backgroundCompletion';
+import type {
+ AgentTriggerContinuePreparation,
+ AgentTriggerExecutionHostDeps,
+} from './triggers/host';
+import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
+import type { AgentTriggerDispatchContext } from './triggers/dispatch';
+import type { AgentTriggerEnqueueOptions } from './triggers/delivery';
+import { BACKGROUND_TOOL_PRODUCER_LEASE_MS } from './backgroundCompletion';
+import { createAgentTriggerEnvelope } from './triggers/envelope';
+import { AgentTriggerExecutionError } from './triggers/host';
+import { truncateMiddle } from '~/utils';
+
+const WAKEUP_ADMISSION_DELAY_MS = 250;
+const MAX_WAKEUP_RESULT_CHARS = 24 * 1024;
+export const BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS: number = 16 * 1024;
+const MESSAGE_SELECT = 'messageId parentMessageId isCreatedByUser createdAt';
+export const BACKGROUND_TOOL_COMPLETION_SOURCE = 'background-tool-completion';
+const EVENT_TYPE = 'background-tool.completion';
+
+export type EnqueueBackgroundToolCompletion = (
+ envelope: unknown,
+ options?: AgentTriggerEnqueueOptions,
+) => Promise<{ deliveryKey: string }>;
+
+export type RetireBackgroundToolCompletion = (
+ deliveryKey: string,
+ sourceId: string,
+ reason: string,
+ options?: BackgroundToolWakeupRetireOptions,
+) => Promise<boolean>;
+
+export type RenewBackgroundToolCompletionProducerLease = (
+ deliveryKey: string,
+ sourceId: string,
+ leaseUntil: Date,
+) => Promise<boolean>;
+
+type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
+ Pick<MessageMethods, 'getMessages'> & {
+ claimBackgroundToolResults(params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskId: string;
+ agentId?: string;
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ limit?: number;
+ }): Promise<BackgroundToolResultClaim>;
+ releaseBackgroundToolResultClaims(params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskIds?: string[];
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ }): Promise<boolean>;
+ getAgentTriggerDeliveryProducerLease(params: {
+ deliveryKey: string;
+ sourceId: string;
+ now: Date;
+ }): Promise<AgentTriggerProducerLeaseStatus>;
+ };
+
+interface GenerationState {
+ status?: unknown;
+ metadata?: {
+ idempotencyClientRequestId?: unknown;
+ terminalPersistencePending?: unknown;
+ };
+}
+
+export interface BackgroundToolCompletionWakeupResolverDeps {
+ methods: WakeupMethods;
+ getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
+}
+
+function executionError(
+ message: string,
+ options: {
+ code: string;
+ retryable: boolean;
+ deferWithoutAttempt?: boolean;
+ status?: number;
+ retryAfter?: string;
+ },
+): AgentTriggerExecutionError {
+ return new AgentTriggerExecutionError(message, {
+ mode: 'continue',
+ certainty: 'definite',
+ ...options,
+ });
+}
+
+function payloadRegistration(
+ envelope: AgentContinueTriggerEnvelope,
+): Pick<BackgroundToolWakeupRegistration, 'taskId' | 'toolCallId' | 'toolName'> | null | undefined {
+ if (
+ envelope.event.source.type !== 'internal' ||
+ envelope.event.source.id !== BACKGROUND_TOOL_COMPLETION_SOURCE ||
+ envelope.event.type !== EVENT_TYPE
+ ) {
+ return;
+ }
+ const payload = envelope.event.payload;
+ if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) {
+ return null;
+ }
+ const { taskId, toolCallId, toolName } = payload;
+ if (
+ typeof taskId !== 'string' ||
+ taskId.length === 0 ||
+ taskId.length > 256 ||
+ typeof toolCallId !== 'string' ||
+ toolCallId.length === 0 ||
+ toolCallId.length > 256 ||
+ typeof toolName !== 'string' ||
+ toolName.length === 0 ||
+ toolName.length > 256
+ ) {
+ return null;
+ }
+ return { taskId, toolCallId, toolName };
+}
+
+function isParentActive(job: GenerationState | null): boolean {
+ return (
+ job?.status === 'running' ||
+ job?.status === 'requires_action' ||
+ job?.metadata?.terminalPersistencePending === true
+ );
+}
+
+function timestamp(message: Pick<IMessage, 'createdAt'>): number {
+ const value = message.createdAt;
+ if (value instanceof Date) {
+ return value.getTime();
+ }
+ const parsed = value == null ? Number.NaN : new Date(value).getTime();
+ return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function latestAssistantDescendant(messages: IMessage[], anchorId: string): string | undefined {
+ const byId = new Map(messages.map((message) => [message.messageId, message]));
+ if (!byId.has(anchorId)) {
+ return;
+ }
+ const memo = new Map<string, boolean>([[anchorId, true]]);
+ const reachesAnchor = (message: IMessage, visiting = new Set<string>()): boolean => {
+ const known = memo.get(message.messageId);
+ if (known != null) {
+ return known;
+ }
+ if (visiting.has(message.messageId)) {
+ memo.set(message.messageId, false);
+ return false;
+ }
+ visiting.add(message.messageId);
+ const parent =
+ typeof message.parentMessageId === 'string' ? byId.get(message.parentMessageId) : undefined;
+ const reachable = parent != null && reachesAnchor(parent, visiting);
+ visiting.delete(message.messageId);
+ memo.set(message.messageId, reachable);
+ return reachable;
+ };
+ const descendants = messages
+ .filter((message) => message.isCreatedByUser === false && reachesAnchor(message))
+ .sort((left, right) => {
+ const time = timestamp(left) - timestamp(right);
+ return time === 0 ? left.messageId.localeCompare(right.messageId) : time;
+ });
+ return descendants[descendants.length - 1]?.messageId;
+}
+
+function fitWakeupResult(output: string, serializedBudget: number): string {
+ if (serializedBudget <= 0 || output.length === 0) {
+ return '';
+ }
+ let low = 0;
+ let high = Math.min(output.length, MAX_WAKEUP_RESULT_CHARS);
+ let fitted = '';
+ while (low <= high) {
+ const limit = Math.floor((low + high) / 2);
+ const candidate = truncateMiddle(output, limit);
+ /** The aggregate limit applies after JSON escaping, not just to raw tool
+ * text. Subtract the empty string's two quote characters because the
+ * fixed payload budget below already includes `result: ""`. */
+ const cost = JSON.stringify(candidate).length - 2;
+ if (cost <= serializedBudget) {
+ fitted = candidate;
+ low = limit + 1;
+ } else {
+ high = limit - 1;
+ }
+ }
+ return fitted;
+}
+
+function buildWakeupInput(
+ results: Array<{
+ taskId: string;
+ toolCallId: string;
+ toolName: string;
+ status: 'completed' | 'error';
+ output: string;
+ }>,
+): string {
+ const header =
+ results.length === 1
+ ? 'A background tool task has finished. Continue using its durable result below.'
+ : `${results.length} background tool tasks have finished. Continue using their durable results below.`;
+ const payload = results.map((result) => ({
+ background_task_id: result.taskId,
+ tool_call_id: result.toolCallId,
+ tool: result.toolName,
+ status: result.status,
+ result: '',
+ }));
+ let remaining = Math.max(
+ 0,
+ BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - header.length - 1 - JSON.stringify(payload).length,
+ );
+ for (let index = 0; index < results.length; index++) {
+ const slots = results.length - index;
+ const share = Math.floor(remaining / slots);
+ const fitted = fitWakeupResult(results[index]?.output ?? '', share);
+ payload[index]!.result = fitted;
+ remaining -= JSON.stringify(fitted).length - 2;
+ }
+ return `${header}\n${JSON.stringify(payload)}`;
+}
+
+/** Resolves a pre-registered delivery only after its result is durably
+ * readable. The message claim elects automatic delivery against manual polls
+ * and returns a bounded sibling batch for the continuation input. */
+export function createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob,
+}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
+ AgentTriggerExecutionHostDeps['prepareContinue']
+> {
+ return async (
+ envelope: AgentContinueTriggerEnvelope,
+ context: AgentTriggerDispatchContext,
+ ): Promise<AgentTriggerContinuePreparation | undefined> => {
+ const registration = payloadRegistration(envelope);
+ if (registration === undefined) {
+ return;
+ }
+ if (registration === null) {
+ throw executionError('The background tool completion payload is invalid.', {
+ code: 'INVALID_BACKGROUND_TOOL_WAKEUP',
+ retryable: false,
+ });
+ }
+ let parentJob: GenerationState | null;
+ try {
+ parentJob = await getGenerationJob(envelope.target.conversationId);
+ } catch (error) {
+ throw executionError(
+ `Parent generation state is temporarily unavailable: ${
+ error instanceof Error ? error.message : String(error)
+ }`,
+ { code: 'PARENT_STATE_UNAVAILABLE', retryable: true },
+ );
+ }
+ if (
+ isParentActive(parentJob) &&
+ parentJob?.metadata?.idempotencyClientRequestId !== context.idempotencyKey
+ ) {
+ throw executionError('The parent generation has not settled yet.', {
+ code: 'PARENT_NOT_READY',
+ retryable: true,
+ status: 409,
+ retryAfter: '1',
+ deferWithoutAttempt: true,
+ });
+ }
+ const userId = envelope.principal.userId;
+ const parent = await methods.getConvo(userId, envelope.target.conversationId);
+ if (parent == null || parent.tenantId !== envelope.principal.tenantId) {
+ throw executionError('The parent conversation is no longer available.', {
+ code: 'PARENT_NOT_FOUND',
+ retryable: false,
+ status: 404,
+ });
+ }
+ const parentMessages = await methods.getMessages(
+ { user: userId, conversationId: envelope.target.conversationId },
+ MESSAGE_SELECT,
+ { sort: { createdAt: 1, _id: 1 } },
+ );
+ const parentMessageId = latestAssistantDescendant(
+ parentMessages,
+ envelope.target.parentMessageId,
+ );
+ if (parentMessageId == null) {
+ throw executionError('The parent conversation branch is no longer available.', {
+ code: 'PARENT_NOT_FOUND',
+ retryable: false,
+ status: 404,
+ });
+ }
+ const claim = await methods.claimBackgroundToolResults({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskId: registration.taskId,
+ agentId: envelope.target.agentId,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ });
+ if (claim.status === 'claimed') {
+ return { status: 'settled' };
+ }
+ if (claim.status !== 'acquired') {
+ let producerLease: AgentTriggerProducerLeaseStatus;
+ try {
+ producerLease = await methods.getAgentTriggerDeliveryProducerLease({
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ now: new Date(),
+ });
+ } catch (error) {
+ throw executionError(
+ `Background tool producer liveness is temporarily unavailable: ${
+ error instanceof Error ? error.message : String(error)
+ }`,
+ { code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
+ );
+ }
+ if (producerLease.status === 'expired') {
+ throw executionError('The process-local background tool executor was lost.', {
+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',
+ retryable: false,
+ });
+ }
+ /** A live lease proves the invocation or its durable persistence retry
+ * still has an owner. Missing remains defer-only for compatibility with
+ * completion rows admitted before producer leases existed. */
+ throw executionError('The background tool result is not durable yet.', {
+ code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
+ retryable: true,
+ status: 409,
+ retryAfter: '1',
+ deferWithoutAttempt: true,
+ });
+ }
+ const taskIds = claim.results.map((result) => result.taskId);
+ const input = buildWakeupInput(claim.results);
+ return {
+ status: 'ready',
+ parentMessageId,
+ input,
+ releaseOnDefiniteFailure: async () => {
+ await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskIds,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ });
+ },
+ };
+ };
+}
+
+/** Pre-registers the ordered completion delivery before external tool work starts. */
+export function createBackgroundToolCompletionWakeupHandler(
+ enqueue: EnqueueBackgroundToolCompletion,
+ retire: RetireBackgroundToolCompletion,
+ renewProducerLease: RenewBackgroundToolCompletionProducerLease,
+): (
+ registration: BackgroundToolWakeupRegistration,
+) => Promise<BackgroundToolWakeupAdmission | false> {
+ return async (registration) => {
+ const parentAgentId = registration.parentAgentId?.trim();
+ if (parentAgentId == null || parentAgentId === '' || isEphemeralAgentId(parentAgentId)) {
+ return false;
+ }
+ const envelope = createAgentTriggerEnvelope({
+ mode: 'continue',
+ requestId: randomUUID(),
+ deliveryId: registration.taskId,
+ receivedAt: Date.now(),
+ principal: {
+ id: registration.userId,
+ ...(registration.tenantId == null ? {} : { tenantId: registration.tenantId }),
+ },
+ event: {
+ id: registration.taskId,
+ type: EVENT_TYPE,
+ occurredAt: registration.createdAt,
+ source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' },
+ payload: {
+ taskId: registration.taskId,
+ toolCallId: registration.toolCallId,
+ toolName: registration.toolName,
+ },
+ },
+ target: {
+ agentId: parentAgentId,
+ conversationId: registration.conversationId,
+ parentMessageId: registration.parentMessageId,
+ },
+ input: 'A background tool task is waiting to complete.',
+ });
+ const admitted = await enqueue(envelope, {
+ /** Pending work gets a task-local lane so a slow tool cannot block an
+ * independently completed sibling. The generation admission fence and
+ * atomic result claim serialize the actual continuations. */
+ orderingKey: `background-tool-completion:${registration.conversationId}:${registration.taskId}`,
+ availableAt: new Date(
+ Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
+ ),
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
+ });
+ return {
+ renew: () =>
+ renewProducerLease(
+ admitted.deliveryKey,
+ BACKGROUND_TOOL_COMPLETION_SOURCE,
+ new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
+ ),
+ retire: (reason, options) =>
+ options == null
+ ? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason)
+ : retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason, options),
+ };
+ };
+}
+
+/** Reopens every result owned by one dead automatic batch. Generation state
+ * first fences an admitted continuation that is still running/finalizing;
+ * delivery retirement then proves no retry remains, and releasing by the
+ * batch-root claim identity makes sibling recovery independent of which task
+ * originally admitted that delivery. */
+export function createBackgroundToolDeadClaimRecovery(
+ retire: RetireBackgroundToolCompletion,
+ releaseClaims: WakeupMethods['releaseBackgroundToolResultClaims'],
+ getGenerationJob: (conversationId: string) => Promise<GenerationState | null | undefined>,
+ fenceGenerationClaim: (input: {
+ userId: string;
+ conversationId: string;
+ claimId: string;
+ }) => Promise<'fenced' | 'started' | 'unavailable'>,
+): BackgroundToolDeadClaimRecovery {
+ return async ({ userId, conversationId, messageId, claimId }) => {
+ const claimGenerationIsActive = async (): Promise<boolean> => {
+ const generation = await getGenerationJob(conversationId);
+ return (
+ generation?.metadata?.idempotencyClientRequestId === claimId && isParentActive(generation)
+ );
+ };
+ if (await claimGenerationIsActive()) {
+ return false;
+ }
+ const retired = await retire(
+ claimId,
+ BACKGROUND_TOOL_COMPLETION_SOURCE,
+ 'dead background completion batch recovered by manual poll',
+ { onlyIfDead: true },
+ );
+ if (!retired) {
+ return false;
+ }
+ /** Retirement closes further delivery retries. The idempotency-claim CAS
+ * closes the remaining claim-to-job-publication window: recovery either
+ * installs a started tombstone that invalidates a delayed creator's token,
+ * or observes that job creation already won. */
+ const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId });
+ if (generationFence === 'unavailable') {
+ return false;
+ }
+ if (await claimGenerationIsActive()) {
+ return false;
+ }
+ return releaseClaims({
+ userId,
+ conversationId,
+ messageId,
+ kind: 'wakeup',
+ claimId,
+ });
+ };
+}
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index 5de6e55ebd..d57a1c964b 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -2,8 +2,8 @@ import { z } from 'zod';
import { logger } from '@librechat/data-schemas';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
+import { backgroundTaskRegistry, CHECK_BACKGROUND_TASK_NAME } from './background';
import { ContentFilterError } from '../middleware/contentFilter';
-import { CHECK_BACKGROUND_TASK_NAME } from './background';
import { createToolExecuteHandler } from './handlers';
interface BatchInput {
@@ -84,6 +84,509 @@ const runBatch = async (
};
describe('createToolExecuteHandler — background tool calls', () => {
+ it('pre-registers an ordinary completion before invoke and persists its terminal receipt', async () => {
+ const events: string[] = [];
+ const retire = jest.fn(async () => true);
+ const preregister = jest.fn(async () => {
+ events.push('preregister');
+ return { renew: jest.fn(async () => true), retire };
+ });
+ const persist = jest.fn(async () => {
+ events.push('persist');
+ return true;
+ });
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(async () => {
+ events.push('invoke');
+ return { content: 'durable result' };
+ }),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister,
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-wakeup',
+ name: tool.name,
+ args: { q: 'continuations', run_in_background: true },
+ stepId: 'step-wakeup',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },
+ });
+
+ expect(events.slice(0, 2)).toEqual(['preregister', 'invoke']);
+ expect(JSON.parse(dispatch.content).message).toContain('host will resume you');
+ await flushMicrotasks();
+ expect(events).toEqual(['preregister', 'invoke', 'persist']);
+ expect(preregister).toHaveBeenCalledWith(
+ expect.objectContaining({
+ toolCallId: 'call-wakeup',
+ conversationId: 'exec_convo',
+ parentMessageId: 'response-1',
+ parentAgentId: 'agent_parent_1',
+ }),
+ );
+ expect(persist).toHaveBeenCalledWith(
+ expect.objectContaining({
+ output: 'durable result',
+ stepId: 'step-wakeup',
+ backgroundTask: expect.objectContaining({
+ taskId: expect.any(String),
+ status: 'completed',
+ }),
+ }),
+ );
+ expect(retire).not.toHaveBeenCalled();
+ });
+
+ it('persists structured background content using the registry serialization', async () => {
+ const structuredContent = [
+ { type: 'text', text: 'structured result' },
+ { type: 'resource', uri: 'memory://result' },
+ ];
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(async () => ({ content: structuredContent })),
+ } as unknown as StructuredToolInterface;
+ const persist = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire: jest.fn(async () => true),
+ })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-structured-wakeup',
+ name: tool.name,
+ args: { q: 'structured', run_in_background: true },
+ stepId: 'step-structured-wakeup',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-structured' },
+ });
+ await flushMicrotasks();
+
+ expect(persist).toHaveBeenCalledWith(
+ expect.objectContaining({ output: JSON.stringify(structuredContent) }),
+ );
+ });
+
+ it('keeps polling guidance when the completion adapter skips registration', async () => {
+ const tool = makeSearchTool({ calls: 0 });
+ const preregister = jest.fn(async () => false as const);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister,
+ persist: jest.fn(async () => true),
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-skipped-wakeup',
+ name: tool.name,
+ args: { q: 'ephemeral', run_in_background: true },
+ stepId: 'step-skipped-wakeup',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },
+ });
+
+ expect(preregister).toHaveBeenCalledTimes(1);
+ expect(JSON.parse(dispatch.content).message).toContain('Call check_background_task');
+ expect(JSON.parse(dispatch.content).message).not.toContain('host will resume you');
+ });
+
+ it('keeps repeated provider ids poll-only when the host step identity is absent', async () => {
+ const tool = makeSearchTool({ calls: 0 });
+ const preregister = jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire: jest.fn(async () => true),
+ }));
+ const persist = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister,
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-without-step',
+ name: tool.name,
+ args: { q: 'legacy', run_in_background: true },
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },
+ });
+ await flushMicrotasks();
+
+ expect(preregister).not.toHaveBeenCalled();
+ expect(persist).not.toHaveBeenCalled();
+ expect(JSON.parse(dispatch.content).message).toContain('Call check_background_task');
+ expect(JSON.parse(dispatch.content).message).not.toContain('host will resume you');
+ });
+
+ it('retires a preregistered delivery when terminal persistence fails', async () => {
+ const tool = makeSearchTool({ calls: 0 });
+ const retire = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
+ persist: jest.fn(async () => false),
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-persistence-failure',
+ name: tool.name,
+ args: { q: 'retire', run_in_background: true },
+ stepId: 'step-persistence-failure',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);
+ });
+
+ it('keeps durable ownership active when an ambiguous persistence failure cannot retire a lease', async () => {
+ const tool = makeSearchTool({ calls: 0 });
+ const retire = jest.fn(async () => false);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
+ persist: jest.fn(async () => {
+ throw new Error('write receipt lost');
+ }),
+ claim: jest.fn(async () => ({ status: 'claimed' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-ambiguous-persistence',
+ name: tool.name,
+ args: { q: 'ambiguous', run_in_background: true },
+ stepId: 'step-ambiguous-persistence',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-ambiguous' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ const taskId = JSON.parse(dispatch.content).background_task_id as string;
+ expect(retire).toHaveBeenCalledWith('background tool result persistence failed', {
+ onlyIfUnclaimed: true,
+ });
+ const task = backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId);
+ expect(task?.completionWakeup).toBe(true);
+ expect(task?.completionPersistenceFailed).toBeUndefined();
+ });
+
+ it('elects live polling only when the settled result actually contains an artifact', async () => {
+ const retire = jest.fn(async () => true);
+ const preregister = jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire,
+ }));
+ const persist = jest.fn(async () => true);
+ const tool = {
+ name: 'artifact_tool',
+ description: 'returns a live artifact',
+ schema: z.object({ q: z.string() }),
+ responseFormat: 'content_and_artifact',
+ invoke: jest.fn(async () => ({
+ content: 'artifact result',
+ artifact: { files: ['report.pdf'] },
+ })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister,
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-artifact-poll',
+ name: tool.name,
+ args: { q: 'artifact', run_in_background: true },
+ stepId: 'step-artifact-poll',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-artifact' },
+ });
+ await flushMicrotasks();
+
+ expect(preregister).toHaveBeenCalledTimes(1);
+ expect(retire).toHaveBeenCalledWith('background tool artifact requires live polling');
+ expect(persist).toHaveBeenCalledWith(
+ expect.objectContaining({
+ backgroundTask: expect.not.objectContaining({ completionWakeup: true }),
+ }),
+ );
+ expect(JSON.parse(dispatch.content).message).toContain('must call check_background_task');
+ expect(JSON.parse(dispatch.content).message).toContain('do not end the turn');
+ });
+
+ it('keeps automatic completion for a declared artifact tool that returns content only', async () => {
+ const retire = jest.fn(async () => true);
+ const persist = jest.fn(async () => true);
+ const tool = {
+ name: 'optional_artifact_tool',
+ description: 'may return an artifact',
+ schema: z.object({ q: z.string() }),
+ responseFormat: 'content_and_artifact',
+ invoke: jest.fn(async () => ({ content: 'content-only result' })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire,
+ })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-optional-artifact',
+ name: tool.name,
+ args: { q: 'content', run_in_background: true },
+ stepId: 'step-optional-artifact',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-optional-artifact' },
+ });
+ await flushMicrotasks();
+
+ expect(retire).not.toHaveBeenCalled();
+ expect(JSON.parse(dispatch.content).message).toContain('must call check_background_task');
+ expect(persist).toHaveBeenCalledWith(
+ expect.objectContaining({
+ backgroundTask: expect.objectContaining({ completionWakeup: true }),
+ }),
+ );
+ });
+
+ it('persists ordinary background failures in the renderer-recognized error shape', async () => {
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(async () => {
+ throw new Error('boom');
+ }),
+ } as unknown as StructuredToolInterface;
+ const persist = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire: jest.fn(async () => true),
+ })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-ordinary-failure',
+ name: tool.name,
+ args: { q: 'fail', run_in_background: true },
+ stepId: 'step-ordinary-failure',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },
+ });
+ await flushMicrotasks();
+
+ expect(persist).toHaveBeenCalledWith(
+ expect.objectContaining({
+ output: 'Error: [search_mcp_docs] tool call failed: boom',
+ backgroundTask: expect.objectContaining({ status: 'error' }),
+ }),
+ );
+ });
+
+ it('persists a terminal receipt when an admitted background task times out', async () => {
+ jest.useFakeTimers({ doNotFake: ['setImmediate'] });
+ try {
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(
+ (_input: unknown, config?: { signal?: AbortSignal }) =>
+ new Promise((_resolve, reject) => {
+ const signal = config?.signal;
+ if (signal?.aborted === true) {
+ reject(signal.reason);
+ return;
+ }
+ signal?.addEventListener('abort', () => reject(signal.reason), { once: true });
+ }),
+ ),
+ } as unknown as StructuredToolInterface;
+ const persist = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire: jest.fn(async () => true),
+ })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-timeout-wakeup',
+ name: tool.name,
+ args: { q: 'hang', run_in_background: true },
+ stepId: 'step-timeout-wakeup',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-timeout' },
+ });
+
+ jest.advanceTimersByTime(31 * 60 * 1000);
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(persist).toHaveBeenCalledWith(
+ expect.objectContaining({
+ output: 'Error: [search_mcp_docs] tool call failed: Background task timed out',
+ backgroundTask: expect.objectContaining({
+ taskId: JSON.parse(dispatch.content).background_task_id,
+ status: 'error',
+ }),
+ }),
+ );
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('keeps an abort-resistant invocation running without a false timeout receipt', async () => {
+ jest.useFakeTimers({ doNotFake: ['setImmediate'] });
+ try {
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(() => new Promise(() => undefined)),
+ } as unknown as StructuredToolInterface;
+ const persist = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire: jest.fn(async () => true),
+ })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-timeout-resistant',
+ name: tool.name,
+ args: { q: 'hang', run_in_background: true },
+ stepId: 'step-timeout-resistant',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-timeout-resistant' },
+ });
+
+ jest.advanceTimersByTime(31 * 60 * 1000);
+ await flushMicrotasks();
+
+ const taskId = JSON.parse(dispatch.content).background_task_id as string;
+ expect(backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId)?.status).toBe('running');
+ expect(persist).not.toHaveBeenCalled();
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
it('persists an Event Actor launch before invoking and terminal evidence before wakeup', async () => {
const events: string[] = [];
let finishTool: ((value: { content: string }) => void) | undefined;
@@ -761,6 +1264,39 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(state.calls).toBe(1);
});
+ it('uses host step identity when repeated provider ids have no turn number', async () => {
+ const state = { calls: 0 } as { calls: number; lastInput?: Record<string, unknown> };
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [makeSearchTool(state)] }),
+ });
+
+ const results = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call_repeated',
+ name: 'search_mcp_docs',
+ args: { q: 'first', run_in_background: true },
+ stepId: 'step-first',
+ },
+ {
+ id: 'call_repeated',
+ name: 'search_mcp_docs',
+ args: { q: 'second', run_in_background: true },
+ stepId: 'step-second',
+ },
+ ],
+ agentId: 'a',
+ configurable: buildConfig(),
+ metadata: { thread_id: 'exec_convo_repeated_steps', run_id: 'response-repeated' },
+ });
+ await flushMicrotasks();
+
+ expect(state.calls).toBe(2);
+ expect(JSON.parse(results[0].content).background_task_id).not.toBe(
+ JSON.parse(results[1].content).background_task_id,
+ );
+ });
+
it('runs the tool synchronously when background is not requested', async () => {
const state = { calls: 0 } as { calls: number; lastInput?: Record<string, unknown> };
const searchTool = makeSearchTool(state);
@@ -1209,6 +1745,66 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
...overrides,
});
+ it('keeps step-less code harvests available to the legacy poll path', async () => {
+ const state: CodeToolState = { calls: 0 };
+ const preregister = jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ retire: jest.fn(async () => true),
+ }));
+ const persistBackgroundCodeResult = jest.fn(async () => ({ attachments: [] }));
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
+ persistBackgroundCodeResult,
+ backgroundToolCompletion: {
+ preregister,
+ persist: jest.fn(async () => true),
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
+ });
+ const configurable = buildConfig(['execute_code']);
+ const metadata = { thread_id: 'exec_convo_step_less_code', run_id: 'response-step-less' };
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [codeCall({ id: 'call_step_less_code', stepId: undefined, turn: undefined })],
+ agentId: 'a',
+ configurable,
+ metadata,
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(preregister).not.toHaveBeenCalled();
+ expect(persistBackgroundCodeResult).toHaveBeenCalledWith(
+ expect.not.objectContaining({ backgroundTask: expect.anything() }),
+ );
+
+ const taskId = JSON.parse(dispatch.content).background_task_id;
+ const [poll] = (await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'poll_step_less_code',
+ name: CHECK_BACKGROUND_TASK_NAME,
+ args: { background_task_id: taskId },
+ },
+ ],
+ agentId: 'a',
+ configurable,
+ metadata: { thread_id: metadata.thread_id, run_id: 'poll-step-less' },
+ })) as Array<{ content: string; artifact?: unknown }>;
+ await flushMicrotasks();
+
+ expect(JSON.parse(poll.content)).toMatchObject({ status: 'completed' });
+ expect(poll.artifact).toEqual(CODE_ARTIFACT);
+ expect(persistBackgroundCodeResult).toHaveBeenCalledTimes(2);
+ expect(persistBackgroundCodeResult).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ reapply: true,
+ toolCallId: 'call_step_less_code',
+ stepId: undefined,
+ }),
+ );
+ });
+
it('carries full code-session config into the detached invoke, harvests onto the dispatch turn, and re-emits on poll', async () => {
const state: CodeToolState = { calls: 0 };
const persistCalls: Array<Record<string, unknown>> = [];
@@ -1305,9 +1901,10 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
type: 'background_task_status',
/** Agent-suffixed: sibling agents' `call_0` markers must not upsert
* over each other client-side. */
- file_id: 'bg-call_code-a',
+ file_id: 'bg-call_code-a-step_1',
messageId: 'msg-dispatch',
toolCallId: 'call_code',
+ stepId: 'step_1',
status: 'completed',
}),
]);
@@ -1549,10 +2146,16 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
});
});
const toolEndCallback = jest.fn();
+ const retire = jest.fn(async () => true);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [codeTool] }),
toolEndCallback,
persistBackgroundCodeResult,
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
+ persist: jest.fn(async () => true),
+ claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ },
});
const configurable = buildConfig(['execute_code']);
const metadata = {
@@ -1569,6 +2172,7 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
const taskId = JSON.parse(dispatch[0].content).background_task_id;
await flushMicrotasks();
await flushMicrotasks();
+ expect(retire).toHaveBeenCalledWith('background code result persistence failed', undefined);
await flushMicrotasks();
for (const pollId of ['call_policy_poll_1', 'call_policy_poll_2']) {
@@ -2040,7 +2644,7 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
expect(JSON.stringify(polled)).not.toContain(detectorRule);
});
- it('re-anchors reaped (timed-out) tasks with the client-recognized failure wrapper', async () => {
+ it('re-anchors abort-confirmed timeout failures with the client-recognized wrapper', async () => {
jest.useFakeTimers({ doNotFake: ['setImmediate'] });
try {
const persistCalls: Array<Record<string, unknown>> = [];
@@ -2048,7 +2652,15 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
name: 'execute_code',
description: 'never settles',
schema: z.object({ lang: z.string(), code: z.string() }),
- invoke: () => new Promise(() => undefined),
+ invoke: (_input: unknown, config?: { signal?: AbortSignal }) =>
+ new Promise((_resolve, reject) => {
+ const signal = config?.signal;
+ if (signal?.aborted === true) {
+ reject(signal.reason);
+ return;
+ }
+ signal?.addEventListener('abort', () => reject(signal.reason), { once: true });
+ }),
} as unknown as StructuredToolInterface;
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [hangingTool] }),
@@ -2066,8 +2678,10 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
metadata: { thread_id: 'exec_convo_reap', run_id: 'msg-reap' },
});
- /** Past the running TTL the registry reaps the never-settling task. */
+ /** Past the deadline the tool acknowledges abort by rejecting. */
jest.advanceTimersByTime(31 * 60 * 1000);
+ await flushMicrotasks();
+ await flushMicrotasks();
const poll = await runBatch(handler, {
toolCalls: [
@@ -2084,10 +2698,13 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
await flushMicrotasks();
expect(JSON.parse(poll[0].content).status).toBe('error');
- const reapply = persistCalls.find((call) => call.reapply === true);
- expect(reapply).toBeDefined();
- expect(String(reapply?.output)).toMatch(/^Error:\s*\[execute_code\]\s*tool call failed:/);
- expect(String(reapply?.output)).toContain('timed out');
+ const timeoutPersistence = persistCalls.find((call) =>
+ String(call.output).includes('timed out'),
+ );
+ expect(timeoutPersistence).toBeDefined();
+ expect(String(timeoutPersistence?.output)).toMatch(
+ /^Error:\s*\[execute_code\]\s*tool call failed:/,
+ );
} finally {
jest.useRealTimers();
}
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index f11d891df3..8bfd1e0c8b 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -17,7 +17,13 @@ import type {
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { CodeEnvRef, PtcToolCallEvent } from 'librechat-data-provider';
import type { ValidationIssue } from '@librechat/data-schemas';
+import type {
+ BackgroundToolDeadClaimRecovery,
+ BackgroundToolWakeupAdmission,
+ BackgroundToolWakeupRegistration,
+} from './backgroundCompletion';
import type { SkillFileRecord, PrimeSkillFilesResult } from './skillFiles';
+import type { BackgroundToolResultState } from './harvest';
import type { CodeExecutionContext } from './execution';
import type { TextContentFragment } from '~/protection';
import type { ServerRequest } from '~/types';
@@ -34,6 +40,7 @@ import {
buildBackgroundHandleContent,
buildBackgroundCapacityContent,
stripBackgroundFromToolDefinitions,
+ withBackgroundTaskTimeout,
BACKGROUND_STATUS_ATTACHMENT_TYPE,
CHECK_BACKGROUND_TASK_NAME,
RUN_IN_BACKGROUND_ARG,
@@ -68,6 +75,7 @@ import {
} from './intent';
import { getSafeErrorMetadata, logAxiosError, runOutsideTracing, truncateMiddle } from '~/utils';
import { resolveCallerCapabilityProjectionSnapshot } from './callerCapabilities';
+import { BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS } from './backgroundCompletion';
import { buildSkillPrimeMessage, SKILL_FILE_PREFIX } from './skills';
import { createSkillContentDigest } from './compatibility';
import { parseFrontmatter } from '../skills/import';
@@ -182,6 +190,7 @@ export interface ToolExecuteOptions {
persistBackgroundCodeResult?: (params: {
toolName: string;
toolCallId: string;
+ stepId?: string;
messageId?: string;
conversationId?: string;
agentId?: string;
@@ -191,7 +200,40 @@ export interface ToolExecuteOptions {
codeExecutionContext?: CodeExecutionContext;
attachments?: unknown[];
reapply?: boolean;
- }) => Promise<{ attachments?: unknown[] } | null>;
+ backgroundTask?: BackgroundToolResultState;
+ resolveBackgroundTask?: () => BackgroundToolResultState;
+ }) => Promise<{ attachments?: unknown[]; deliveryReady?: boolean } | null>;
+ /** Shared ordinary-tool completion lifecycle. The delivery is registered
+ * before invoke; settlement is persisted onto the original response row. */
+ backgroundToolCompletion?: {
+ preregister?: (
+ registration: BackgroundToolWakeupRegistration,
+ ) => Promise<BackgroundToolWakeupAdmission | false>;
+ persist: (params: {
+ toolName: string;
+ toolCallId: string;
+ stepId?: string;
+ messageId?: string;
+ conversationId?: string;
+ agentId?: string;
+ output?: string;
+ backgroundTask: BackgroundToolResultState;
+ resolveBackgroundTask?: () => BackgroundToolResultState;
+ }) => Promise<boolean>;
+ claim: (params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskId: string;
+ agentId?: string;
+ kind: 'manual';
+ claimId: string;
+ }) => Promise<
+ | { status: 'acquired' | 'not_found' | 'not_ready' }
+ | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }
+ >;
+ recoverDeadClaim?: BackgroundToolDeadClaimRecovery;
+ };
/** Emits an `attachment` SSE event on the current request's live stream. */
emitAttachment?: (attachment: unknown) => void;
/**
@@ -4157,7 +4199,7 @@ function getFileAuthoringQueueKey(
* so wrap them identically before patching the dispatch row, or a reloaded
* failed background run renders as clean stdout.
*/
-function toCodeToolFailure(toolName: string, message: string): string {
+function toBackgroundToolFailure(toolName: string, message: string): string {
if (/^Error:\s*(\[.*?\]\s*)*tool call failed:/i.test(message)) {
return message;
}
@@ -4270,6 +4312,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
toolEndCallback,
eventActorDetachedAction,
persistBackgroundCodeResult,
+ backgroundToolCompletion,
emitAttachment,
emitPtcProgress,
subagentTasks,
@@ -4426,6 +4469,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
const isCodeCall = isCodeSessionAwareToolCall(tc.name, mergedConfigurable);
const harvestEnabled = isCodeCall && persistBackgroundCodeResult != null;
+ const liveArtifactPollRequired =
+ !harvestEnabled &&
+ (tool as StructuredToolInterface & { responseFormat?: unknown }).responseFormat ===
+ Constants.CONTENT_AND_ARTIFACT;
+ const backgroundStepId =
+ typeof tc.stepId === 'string' && tc.stepId.trim() !== '' ? tc.stepId : undefined;
const strippedArgs = stripIntentForInvoke(stripRunInBackgroundArg(tc.args), tool);
const normalizedArgs = normalizeToolInvokeArgs(strippedArgs, tool);
const filtered = filteredToolArgumentsResult(tc, backgroundReq, normalizedArgs);
@@ -4436,14 +4485,16 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
userId: backgroundUserId,
conversationId: backgroundConversationId,
toolCallId: tc.id,
+ stepId: backgroundStepId,
toolName: tc.name,
messageId: backgroundRunId,
harvestStarted: harvestEnabled,
+ liveArtifactPollRequired,
/** Scope idempotency to the agent + run + turn so a later turn's
* or a second agent's repeated provider id (e.g. `call_0`)
* starts a fresh task instead of colliding. */
agentId,
- runId: `${backgroundRunId ?? ''}:${tc.turn ?? ''}`,
+ runId: `${backgroundRunId ?? ''}:${tc.turn ?? backgroundStepId ?? ''}`,
};
const capacityAdmission =
eventActorDetachedAction == null
@@ -4538,7 +4589,48 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
};
}
const { task, isNew } = created;
+ let completionPreregistered = task.completionWakeup === true;
+ let completionAdmission: BackgroundToolWakeupAdmission | undefined;
if (isNew) {
+ if (
+ detachedReservation?.status !== 'reserved' &&
+ backgroundToolCompletion?.preregister != null &&
+ backgroundStepId != null &&
+ backgroundRunId != null &&
+ backgroundRunId !== ''
+ ) {
+ try {
+ const admission = await backgroundToolCompletion.preregister({
+ taskId: task.id,
+ toolCallId: tc.id,
+ toolName: tc.name,
+ userId: backgroundUserId,
+ ...(typeof backgroundReq?.user?.tenantId === 'string' &&
+ backgroundReq.user.tenantId !== ''
+ ? { tenantId: backgroundReq.user.tenantId }
+ : {}),
+ conversationId: backgroundConversationId,
+ parentMessageId: backgroundRunId,
+ parentAgentId: agentId,
+ createdAt: task.createdAt,
+ });
+ if (admission !== false) {
+ completionAdmission = admission;
+ completionPreregistered = true;
+ backgroundTaskRegistry.markCompletionWakeup(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ admission,
+ );
+ }
+ } catch (registrationError) {
+ logger.warn(
+ `[background] Failed to preregister completion for task ${task.id}; polling remains available.`,
+ registrationError,
+ );
+ }
+ }
/** Persists the settled result onto the dispatch turn's message
* (patch the tool-call part's output, persist generated files,
* append attachments), so a backgrounded code call reads like a
@@ -4547,79 +4639,217 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
* the dispatch row may not exist until that turn finalizes, so
* gating `complete()` on the patch would livelock same-turn
* polls on `running`. Failures degrade to poll-only delivery. */
- const harvestCodeResult = (params: {
+ const persistBackgroundResult = async (params: {
output?: string;
artifact?: unknown;
- }): void => {
- if (!harvestEnabled || !persistBackgroundCodeResult) {
+ status: 'completed' | 'error';
+ }): Promise<void> => {
+ /** A provider id alone is not a durable part identity: it may
+ * repeat in later turns of the same response. New automatic
+ * completion delivery therefore fails closed to the legacy
+ * poll path when the host run-step anchor is unavailable. */
+ if (
+ detachedReservation?.status !== 'reserved' &&
+ backgroundToolCompletion != null &&
+ backgroundStepId == null &&
+ !harvestEnabled
+ ) {
return;
}
- void (async () => {
+ const resolveBackgroundTask = (): BackgroundToolResultState => {
+ const current = backgroundTaskRegistry.get(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ return {
+ taskId: task.id,
+ toolName: tc.name,
+ status: params.status,
+ settledAt: new Date(current?.updatedAt ?? Date.now()),
+ ...(completionPreregistered ? { completionWakeup: true } : {}),
+ ...(current?.resultClaim != null
+ ? {
+ resultClaim: {
+ kind: current.resultClaim.kind,
+ claimId: current.resultClaim.claimId,
+ claimedAt: new Date(current.resultClaim.claimedAt),
+ },
+ }
+ : {}),
+ };
+ };
+ const localTask = backgroundTaskRegistry.get(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ const backgroundTask = resolveBackgroundTask();
+ const retireFailedPersistence = async (
+ reason: string,
+ certainty: 'definite' | 'ambiguous',
+ ): Promise<void> => {
+ if (completionAdmission == null) {
+ backgroundTaskRegistry.markCompletionPersistenceFailed(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ return;
+ }
+ try {
+ /** A thrown write receipt is ambiguous: Mongo may have
+ * applied it before the response was lost, so only an
+ * unclaimed delivery may fall back. A returned `false`
+ * proves no terminal row was anchored and may retire a
+ * live deferring lease before it dead-letters forever. */
+ const retired = await completionAdmission.retire(
+ reason,
+ certainty === 'ambiguous' ? { onlyIfUnclaimed: true } : undefined,
+ );
+ if (!retired) {
+ logger.warn(
+ `[background] Could not retire failed completion delivery for task ${task.id}.`,
+ );
+ return;
+ }
+ backgroundTaskRegistry.markCompletionPersistenceFailed(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ } catch (retireError) {
+ logger.warn(
+ `[background] Failed to retire completion delivery for task ${task.id}:`,
+ retireError,
+ );
+ }
+ };
+ if (!harvestEnabled || !persistBackgroundCodeResult) {
+ if (
+ backgroundToolCompletion == null ||
+ detachedReservation?.status === 'reserved'
+ ) {
+ return;
+ }
try {
- const persisted = await persistBackgroundCodeResult({
+ const deliveryReady = await backgroundToolCompletion.persist({
toolName: tc.name,
toolCallId: tc.id,
+ stepId: backgroundStepId,
messageId: backgroundRunId,
conversationId: backgroundConversationId,
- /** Disambiguates repeated provider ids (e.g. `call_0`)
- * across agents sharing one response message. */
agentId,
- /** Stale-output ordering is decided by DISPATCH order,
- * not harvest wall-clock: a slow old task settling
- * after a newer run wrote the same filename must not
- * overwrite it. */
- dispatchedAt: task.createdAt,
- codeExecutionContext,
- ...params,
+ output: params.output ?? localTask?.result,
+ backgroundTask,
+ resolveBackgroundTask,
});
- if (persisted == null) {
- /** Harvest never persisted anything (missing anchor
- * identity): hand delivery back to the legacy poll-turn
- * callback, restoring the artifact if a poll already
- * claimed it while the harvest was in flight. */
- backgroundTaskRegistry.revokeHarvest(
- backgroundUserId,
- backgroundConversationId,
- task.id,
- params.artifact,
+ if (!deliveryReady) {
+ await retireFailedPersistence(
+ 'background tool result was not persisted',
+ 'definite',
);
- return;
}
- backgroundTaskRegistry.finishHarvest(
+ } catch (persistError) {
+ await retireFailedPersistence(
+ 'background tool result persistence failed',
+ isContentFilterError(persistError) ? 'definite' : 'ambiguous',
+ );
+ logger.warn(
+ `[background] Failed to persist result for task ${task.id}:`,
+ persistError,
+ );
+ }
+ return;
+ }
+ try {
+ const persisted = await persistBackgroundCodeResult({
+ toolName: tc.name,
+ toolCallId: tc.id,
+ stepId: backgroundStepId,
+ messageId: backgroundRunId,
+ conversationId: backgroundConversationId,
+ /** Disambiguates repeated provider ids (e.g. `call_0`)
+ * across agents sharing one response message. */
+ agentId,
+ /** Stale-output ordering is decided by DISPATCH order,
+ * not harvest wall-clock: a slow old task settling
+ * after a newer run wrote the same filename must not
+ * overwrite it. */
+ dispatchedAt: task.createdAt,
+ codeExecutionContext,
+ ...(detachedReservation?.status === 'reserved' || !completionPreregistered
+ ? {}
+ : { backgroundTask, resolveBackgroundTask }),
+ output: params.output ?? localTask?.result,
+ artifact: params.artifact,
+ });
+ if (persisted == null) {
+ /** Harvest never persisted anything (missing anchor
+ * identity): hand delivery back to the legacy poll-turn
+ * callback, restoring the artifact if a poll already
+ * claimed it while the harvest was in flight. */
+ backgroundTaskRegistry.revokeHarvest(
backgroundUserId,
backgroundConversationId,
task.id,
- persisted.attachments,
+ params.artifact,
);
- } catch (persistError) {
- if (isContentFilterError(persistError)) {
- backgroundTaskRegistry.blockArtifact(
- backgroundUserId,
- backgroundConversationId,
- task.id,
- persistError instanceof ContentFilterError
- ? modelBoundContentFilterErrorMessage(persistError.body)
- : persistError.body.message,
- );
- logger.warn(
- `[background] Generated code output for task ${task.id} was blocked by content policy.`,
+ if (completionPreregistered) {
+ await retireFailedPersistence(
+ 'background code result had no durable message anchor',
+ 'definite',
);
- return;
}
- logger.warn(
- `[background] Failed to persist code result for task ${task.id}:`,
- persistError,
+ return;
+ }
+ if (persisted.deliveryReady === false) {
+ await retireFailedPersistence(
+ 'background code result was not persisted',
+ 'definite',
);
- backgroundTaskRegistry.revokeHarvest(
+ }
+ backgroundTaskRegistry.finishHarvest(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ persisted.attachments,
+ );
+ } catch (persistError) {
+ if (completionPreregistered) {
+ await retireFailedPersistence(
+ 'background code result persistence failed',
+ isContentFilterError(persistError) ? 'definite' : 'ambiguous',
+ );
+ }
+ if (isContentFilterError(persistError)) {
+ backgroundTaskRegistry.blockArtifact(
backgroundUserId,
backgroundConversationId,
task.id,
- params.artifact,
+ persistError instanceof ContentFilterError
+ ? modelBoundContentFilterErrorMessage(persistError.body)
+ : persistError.body.message,
);
+ logger.warn(
+ `[background] Generated code output for task ${task.id} was blocked by content policy.`,
+ );
+ return;
}
- })();
+ logger.warn(
+ `[background] Failed to persist code result for task ${task.id}:`,
+ persistError,
+ );
+ backgroundTaskRegistry.revokeHarvest(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ params.artifact,
+ );
+ }
};
let invokePromise: Promise<{ content?: unknown; artifact?: unknown }>;
+ const backgroundAbortController = new AbortController();
try {
invokePromise = Promise.resolve(
tool.invoke(normalizedArgs, {
@@ -4628,6 +4858,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
* `_runtime_session_hint` or it runs fileless on the
* Code API's default runtime session. */
toolCall: buildToolCallConfig(tc, mergedConfigurable),
+ signal: backgroundAbortController.signal,
configurable: {
...mergedConfigurable,
...(detachedReservation?.status === 'reserved'
@@ -4684,9 +4915,42 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
);
}
};
+ let producerHeartbeatInFlight: Promise<void> | undefined;
+ const producerAdmission = completionAdmission;
+ const producerHeartbeat =
+ producerAdmission == null
+ ? undefined
+ : setInterval(() => {
+ if (producerHeartbeatInFlight != null) {
+ return;
+ }
+ producerHeartbeatInFlight = producerAdmission
+ .renew()
+ .then((renewed) => {
+ if (!renewed) {
+ logger.warn(
+ `[background] Completion producer lease was not renewed for task ${task.id}.`,
+ );
+ }
+ })
+ .catch((heartbeatError) => {
+ logger.warn(
+ `[background] Failed to renew completion producer lease for task ${task.id}:`,
+ heartbeatError,
+ );
+ })
+ .finally(() => {
+ producerHeartbeatInFlight = undefined;
+ });
+ }, BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS);
+ (producerHeartbeat as { unref?: () => void } | undefined)?.unref?.();
void (async () => {
try {
- const result = await invokePromise;
+ const result = await withBackgroundTaskTimeout(invokePromise, () =>
+ backgroundAbortController.abort(
+ new DOMException('Background task timed out', 'AbortError'),
+ ),
+ );
if (isCodeCall) {
markCodeSandboxWarm();
}
@@ -4701,13 +4965,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
if (filteredOutput != null) {
const policyError =
filteredOutput.errorMessage ?? 'Submitted content was blocked.';
- const errorOutput = isCodeCall
- ? toCodeToolFailure(tc.name, policyError)
- : policyError;
+ const errorOutput = toBackgroundToolFailure(tc.name, policyError);
+ const registryError = isCodeCall ? errorOutput : policyError;
if (
!(await persistDetachedTerminal({
status: 'succeeded',
- result: errorOutput,
+ result: registryError,
}))
) {
return;
@@ -4716,10 +4979,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
backgroundUserId,
backgroundConversationId,
task.id,
- errorOutput,
+ registryError,
{ harvestStarted: harvestEnabled },
);
- harvestCodeResult({ output: errorOutput });
+ await persistBackgroundResult({ output: errorOutput, status: 'error' });
await wakeDetachedActor();
return;
}
@@ -4738,15 +5001,51 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
) {
return;
}
- backgroundTaskRegistry.complete(
+ if (result.artifact != null && !harvestEnabled && completionAdmission != null) {
+ /** Eligibility is decided from the actual result, not the
+ * tool's declared response format. A content-only result
+ * from a content-and-artifact tool can wake normally; an
+ * actual artifact still needs the live poll callback. */
+ try {
+ const retired = await completionAdmission.retire(
+ 'background tool artifact requires live polling',
+ );
+ if (!retired) {
+ logger.warn(
+ `[background] Could not retire artifact wakeup for task ${task.id}.`,
+ );
+ }
+ } catch (retireError) {
+ logger.warn(
+ `[background] Failed to retire artifact wakeup for task ${task.id}:`,
+ retireError,
+ );
+ } finally {
+ /** Never publish an eligibility marker for an artifact
+ * the continuation cannot reconstruct. An ambiguous
+ * retire receipt therefore fails closed to polling; any
+ * surviving delivery expires with the producer lease. */
+ completionPreregistered = false;
+ backgroundTaskRegistry.markCompletionPersistenceFailed(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ }
+ }
+ const storedContent = backgroundTaskRegistry.complete(
backgroundUserId,
backgroundConversationId,
task.id,
{ content, artifact: result.artifact, harvestStarted: harvestEnabled },
);
- harvestCodeResult({
- output: typeof content === 'string' ? content : undefined,
+ await persistBackgroundResult({
+ /** Use the registry's canonical bounded serialization so
+ * structured content cannot leave the durable card on its
+ * synthetic running handle. */
+ output: storedContent,
artifact: result.artifact,
+ status: 'completed',
});
await wakeDetachedActor();
} catch (toolError) {
@@ -4755,8 +5054,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
? modelBoundContentFilterErrorMessage(toolError.body)
: null;
const { message } = getSafeToolError(toolError);
- const errorOutput =
- policyError ?? (isCodeCall ? toCodeToolFailure(tc.name, message) : message);
+ const errorOutput = policyError ?? message;
const filteredError =
policyError == null
? filteredToolOutputResult(tc, backgroundReq, {
@@ -4764,10 +5062,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
})
: null;
const neutralizedError = filteredError?.errorMessage ?? errorOutput;
- const deliveredError =
- isCodeCall && (policyError != null || filteredError != null)
- ? toCodeToolFailure(tc.name, neutralizedError)
- : neutralizedError;
+ const deliveredError = toBackgroundToolFailure(tc.name, neutralizedError);
+ const registryError = isCodeCall ? deliveredError : neutralizedError;
const detachedTerminalStatus =
toolError instanceof Error &&
(toolError.name === 'AbortError' ||
@@ -4777,7 +5073,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
if (
!(await persistDetachedTerminal({
status: detachedTerminalStatus,
- error: deliveredError,
+ error: registryError,
}))
) {
return;
@@ -4786,14 +5082,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
backgroundUserId,
backgroundConversationId,
task.id,
- deliveredError,
+ registryError,
/** Failed code tasks join the heal path too: without this,
* a full-row save reverting the error patch would leave
* the dispatch card on the handle JSON forever. */
{ harvestStarted: harvestEnabled },
);
- harvestCodeResult({ output: deliveredError });
+ await persistBackgroundResult({ output: deliveredError, status: 'error' });
await wakeDetachedActor();
+ } finally {
+ if (producerHeartbeat != null) {
+ clearInterval(producerHeartbeat);
+ }
+ await producerHeartbeatInFlight;
}
})();
if (
@@ -4810,7 +5111,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
return {
toolCallId: tc.id,
status: 'success' as const,
- content: buildBackgroundHandleContent(task),
+ content: buildBackgroundHandleContent(task, {
+ completionWakeup: completionPreregistered,
+ liveArtifactPollRequired,
+ }),
};
};
@@ -4846,6 +5150,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
agentId,
runId: `${backgroundRunId ?? ''}:${tc.turn ?? ''}`,
subagentTasks,
+ claimBackgroundToolResult: backgroundToolCompletion?.claim,
+ recoverDeadBackgroundToolClaim: backgroundToolCompletion?.recoverDeadClaim,
});
const taskSnapshot = getBackgroundTaskSnapshot({
userId: backgroundUserId,
@@ -4991,16 +5297,17 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
try {
emitAttachment({
type: BACKGROUND_STATUS_ATTACHMENT_TYPE,
- /** Provider ids repeat across agents in handoffs;
- * the agent suffix keeps sibling markers from
+ /** Provider ids repeat across agents and turns; the
+ * host identity suffix keeps sibling markers from
* upserting over each other client-side. */
file_id: `bg-${delivery.toolCallId}${
delivery.agentId != null ? `-${delivery.agentId}` : ''
- }`,
+ }${delivery.stepId != null ? `-${delivery.stepId}` : ''}`,
messageId: delivery.messageId,
conversationId: backgroundConversationId,
toolCallId: delivery.toolCallId,
agentId: delivery.agentId,
+ stepId: delivery.stepId,
status: delivery.status,
});
} catch (emitError) {
@@ -5012,12 +5319,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
if (persistBackgroundCodeResult && delivery.messageId) {
/** Error tasks carry their message in `error`, not
- * `result`; reaped (timed-out) tasks store it raw, so
- * wrap here — `toCodeToolFailure` is a no-op for
+ * `result`; abort-confirmed timeouts store it raw, so
+ * wrap here — `toBackgroundToolFailure` is a no-op for
* already-wrapped detached failures. */
const reapplyOutput =
delivery.status === 'error'
- ? toCodeToolFailure(
+ ? toBackgroundToolFailure(
delivery.toolName,
delivery.error ?? delivery.result ?? 'Background task failed',
)
@@ -5025,6 +5332,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
void persistBackgroundCodeResult({
toolName: delivery.toolName,
toolCallId: delivery.toolCallId,
+ stepId: delivery.stepId,
messageId: delivery.messageId,
conversationId: backgroundConversationId,
agentId: delivery.agentId,
diff --git a/packages/api/src/agents/harvest.spec.ts b/packages/api/src/agents/harvest.spec.ts
index dd588e445a..dde304283f 100644
--- a/packages/api/src/agents/harvest.spec.ts
+++ b/packages/api/src/agents/harvest.spec.ts
@@ -1,5 +1,5 @@
import type { ServerRequest } from '~/types';
-import { createBackgroundCodeResultHandler } from './harvest';
+import { createBackgroundCodeResultHandler, createBackgroundToolResultHandler } from './harvest';
const req = {
user: { id: 'user-1' },
@@ -8,6 +8,7 @@ const req = {
const params = {
toolName: 'execute_code',
toolCallId: 'tool-call-1',
+ stepId: 'step-1',
messageId: 'message-1',
conversationId: 'conversation-1',
output: 'safe output',
@@ -47,7 +48,7 @@ describe('createBackgroundCodeResultHandler generated-file preflight', () => {
});
await expect(handler(params)).resolves.toEqual({
- attachments: [{ file_id: 'persisted-file' }],
+ attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }],
});
expect(preflightCodeOutputBatch).toHaveBeenCalledWith({
@@ -72,7 +73,8 @@ describe('createBackgroundCodeResultHandler generated-file preflight', () => {
expect(updateToolCallResult).toHaveBeenCalledWith(
expect.objectContaining({
output: 'safe output',
- attachments: [{ file_id: 'persisted-file' }],
+ stepId: 'step-1',
+ attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }],
}),
);
});
@@ -96,3 +98,56 @@ describe('createBackgroundCodeResultHandler generated-file preflight', () => {
expect(updateToolCallResult).not.toHaveBeenCalled();
});
});
+
+describe('createBackgroundToolResultHandler claim ownership', () => {
+ it('re-reads a same-generation manual claim before each persistence retry', async () => {
+ let claimed = false;
+ const updateToolCallResult = jest
+ .fn()
+ .mockImplementationOnce(async () => {
+ claimed = true;
+ return { matched: false, unfinished: false };
+ })
+ .mockResolvedValueOnce({ matched: true, unfinished: false });
+ const handler = createBackgroundToolResultHandler({ req, updateToolCallResult });
+ const baseState = {
+ taskId: 'task-1',
+ toolName: 'slow_tool',
+ status: 'completed' as const,
+ settledAt: new Date('2026-08-30T00:00:00Z'),
+ };
+
+ await expect(
+ handler({
+ toolName: 'slow_tool',
+ toolCallId: 'call-1',
+ stepId: 'step-1',
+ messageId: 'message-1',
+ conversationId: 'conversation-1',
+ output: 'done',
+ backgroundTask: baseState,
+ resolveBackgroundTask: () => ({
+ ...baseState,
+ ...(claimed
+ ? {
+ resultClaim: {
+ kind: 'manual' as const,
+ claimId: 'poll-1',
+ claimedAt: new Date('2026-08-30T00:00:01Z'),
+ },
+ }
+ : {}),
+ }),
+ }),
+ ).resolves.toBe(true);
+
+ expect(updateToolCallResult).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({
+ backgroundTask: expect.objectContaining({
+ resultClaim: expect.objectContaining({ kind: 'manual', claimId: 'poll-1' }),
+ }),
+ }),
+ );
+ });
+});
diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts
index b121cdbd95..78259f5e4e 100644
--- a/packages/api/src/agents/harvest.ts
+++ b/packages/api/src/agents/harvest.ts
@@ -12,7 +12,6 @@ import type { ServerRequest } from '~/types';
const BACKGROUND_PATCH_RETRY_DELAYS_MS = [
250, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000, 120_000, 180_000, 240_000, 300_000,
];
-
interface HarvestFileRef {
id: string;
name: string;
@@ -31,6 +30,20 @@ export interface ProcessedCodeOutput {
previewRevision?: number;
}
+export interface BackgroundToolResultState {
+ taskId: string;
+ toolName: string;
+ status: 'completed' | 'error';
+ settledAt: Date;
+ /** This exact task owns a pre-registered automatic continuation delivery. */
+ completionWakeup?: true;
+ resultClaim?: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ claimedAt: Date;
+ };
+}
+
export interface CodeHarvestDeps {
req: ServerRequest;
/** Data-schemas method: idempotent tool-call part patch + attachment append. */
@@ -39,10 +52,12 @@ export interface CodeHarvestDeps {
messageId: string;
conversationId: string;
toolCallId: string;
+ stepId?: string;
agentId?: string;
output?: string;
attachments?: unknown[];
markBackgrounded?: boolean;
+ backgroundTask?: BackgroundToolResultState;
}) => Promise<{ matched: boolean; unfinished: boolean }>;
/** Host preflight: inspects the entire generated-file batch before any write. */
preflightCodeOutputBatch: (params: {
@@ -77,6 +92,7 @@ export interface CodeHarvestDeps {
export interface CodeHarvestParams {
toolName: string;
toolCallId: string;
+ stepId?: string;
messageId?: string;
conversationId?: string;
/** Dispatching agent — scopes the part patch when provider tool-call ids
@@ -91,14 +107,86 @@ export interface CodeHarvestParams {
codeExecutionContext?: CodeExecutionContext;
attachments?: unknown[];
reapply?: boolean;
+ backgroundTask?: BackgroundToolResultState;
+ /** Re-reads local claim ownership on every retry so a same-generation
+ * manual poll cannot be overwritten by a later automatic continuation. */
+ resolveBackgroundTask?: () => BackgroundToolResultState;
}
export type CodeHarvestHandler = (
params: CodeHarvestParams,
-) => Promise<{ attachments: unknown[] } | null>;
+) => Promise<{ attachments: unknown[]; deliveryReady?: boolean } | null>;
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+async function persistBackgroundToolResultRow(
+ updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
+ params: {
+ userId: string;
+ messageId: string;
+ conversationId: string;
+ toolCallId: string;
+ stepId?: string;
+ agentId?: string;
+ output?: string;
+ attachments?: unknown[];
+ backgroundTask?: BackgroundToolResultState;
+ resolveBackgroundTask?: () => BackgroundToolResultState;
+ },
+): Promise<boolean> {
+ const { resolveBackgroundTask, ...persistedParams } = params;
+ for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) {
+ const currentBackgroundTask = resolveBackgroundTask?.() ?? persistedParams.backgroundTask;
+ const result = await updateToolCallResult({
+ ...persistedParams,
+ ...(currentBackgroundTask == null ? {} : { backgroundTask: currentBackgroundTask }),
+ markBackgrounded: true,
+ });
+ if (result.matched && !result.unfinished) {
+ return true;
+ }
+ if (attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length) {
+ break;
+ }
+ await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]);
+ }
+ return false;
+}
+
+/** Persists an ordinary detached tool result without invoking code-artifact processing. */
+export function createBackgroundToolResultHandler(
+ deps: Pick<CodeHarvestDeps, 'req' | 'updateToolCallResult'>,
+): (params: Omit<CodeHarvestParams, 'artifact' | 'codeExecutionContext'>) => Promise<boolean> {
+ return async ({
+ toolCallId,
+ stepId,
+ messageId,
+ conversationId,
+ agentId,
+ output,
+ attachments,
+ backgroundTask,
+ resolveBackgroundTask,
+ }) => {
+ const userId = deps.req.user?.id;
+ if (!userId || !messageId || !conversationId || backgroundTask == null) {
+ return false;
+ }
+ return persistBackgroundToolResultRow(deps.updateToolCallResult, {
+ userId,
+ messageId,
+ conversationId,
+ toolCallId,
+ stepId,
+ agentId,
+ output,
+ attachments,
+ backgroundTask,
+ resolveBackgroundTask,
+ });
+ };
+}
+
/**
* Handles a backgrounded code-execution result once the detached call settles:
* persists generated files (same `processCodeOutput` path as the foreground
@@ -125,6 +213,7 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
} = deps;
return async ({
toolCallId,
+ stepId,
messageId,
conversationId,
agentId,
@@ -134,6 +223,8 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
codeExecutionContext,
attachments: knownAttachments,
reapply,
+ backgroundTask,
+ resolveBackgroundTask,
}) => {
const userId = req.user?.id;
if (!userId || !messageId || !conversationId) {
@@ -141,17 +232,20 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
}
if (reapply === true) {
+ const currentBackgroundTask = resolveBackgroundTask?.() ?? backgroundTask;
const reapplied = await updateToolCallResult({
userId,
messageId,
conversationId,
toolCallId,
+ stepId,
agentId,
output,
attachments: knownAttachments ?? [],
/** The heal path must re-stamp the marker too: the full-row save it
* repairs reverted the whole patched part, marker included. */
markBackgrounded: true,
+ ...(currentBackgroundTask != null ? { backgroundTask: currentBackgroundTask } : {}),
});
if (!reapplied.matched) {
logger.debug(
@@ -192,7 +286,8 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
downloadFallback,
});
if (result?.file) {
- attachments.push(result.file);
+ const anchoredFile = stepId == null ? result.file : { ...result.file, stepId };
+ attachments.push(anchoredFile);
/** No live stream at completion time; the client's preview polling
* (or the poll turn's re-emit) surfaces the finalized preview. */
runPreviewFinalize({
@@ -206,40 +301,24 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
}
}
- let patched = false;
- for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) {
- const result = await updateToolCallResult({
- userId,
- messageId,
- conversationId,
- toolCallId,
- agentId,
- output,
- attachments,
- /** This patch replaces the dispatch-handle output — the client's only
- * transient signal that the call ran detached — so it persists the
- * durable `backgrounded` marker in the same atomic write. */
- markBackgrounded: true,
- });
- patched = result.matched;
- /** An `unfinished` match is a mid-turn partial save (client disconnect):
- * the eventual finalize overwrites it with in-memory content — the
- * handle JSON — so keep re-applying (idempotent) until a finalized row
- * holds the patch. */
- if (
- (result.matched && !result.unfinished) ||
- attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length
- ) {
- break;
- }
- await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]);
- }
- if (!patched) {
+ const deliveryReady = await persistBackgroundToolResultRow(updateToolCallResult, {
+ userId,
+ messageId,
+ conversationId,
+ toolCallId,
+ stepId,
+ agentId,
+ output,
+ attachments,
+ ...(backgroundTask != null ? { backgroundTask } : {}),
+ ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
+ });
+ if (!deliveryReady) {
logger.warn(
`[background] Could not anchor code result onto message ${messageId} (tool call ${toolCallId}); ` +
'the dispatch turn never persisted. Poll delivery still returns the result.',
);
}
- return { attachments };
+ return { attachments, ...(backgroundTask != null ? { deliveryReady } : {}) };
};
}
diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts
index 49ea87f114..3f3d9cfa95 100644
--- a/packages/api/src/agents/index.ts
+++ b/packages/api/src/agents/index.ts
@@ -20,6 +20,8 @@ export * from './execution';
export * from './handlers';
export * from './guard';
export * from './harvest';
+export * from './backgroundCompletion';
+export * from './backgroundCompletionWakeup';
export * from './initialize';
export * from './legacy';
export * from './lazySubagents';
diff --git a/packages/api/src/agents/triggers/delivery.ts b/packages/api/src/agents/triggers/delivery.ts
index 4920033a47..1eb96ea30e 100644
--- a/packages/api/src/agents/triggers/delivery.ts
+++ b/packages/api/src/agents/triggers/delivery.ts
@@ -24,6 +24,8 @@ export interface AgentTriggerEnqueueOptions {
/** Server-owned capability fence. Deliveries carrying this marker remain
* invisible to older workers during a rolling deployment. */
requiredWorkerCapability?: string;
+ /** Private liveness lease for process-owned work behind a capability fence. */
+ producerLeaseUntil?: Date;
}
export interface PreparedAgentTriggerDelivery {
@@ -42,6 +44,7 @@ export interface PreparedAgentTriggerDelivery {
* admitted child turn records an authoritative terminal outcome. */
awaitTerminalHandling?: boolean;
requiredWorkerCapability?: string;
+ producerLeaseUntil?: Date;
}
export class AgentTriggerDeliveryError extends TypeError {
@@ -80,6 +83,16 @@ function requireAvailableAt(value: Date | undefined): Date {
return new Date(availableAt);
}
+function optionalDate(value: Date | undefined, field: string): Date | undefined {
+ if (value == null) {
+ return;
+ }
+ if (!(value instanceof Date) || !Number.isFinite(value.getTime())) {
+ throw new AgentTriggerDeliveryError(`${field} must be a valid Date`);
+ }
+ return new Date(value);
+}
+
function coalescingIdentity(
envelope: AgentTriggerEnvelope,
options: AgentTriggerCoalesceOptions | undefined,
@@ -160,6 +173,10 @@ export function prepareAgentTriggerDelivery(
// a fresh request trace without weakening content-conflict detection.
const { requestId: _requestId, receivedAt: _receivedAt, ...durableIdentity } = envelope;
const requestedAvailableAt = requireAvailableAt(options.availableAt);
+ const producerLeaseUntil = optionalDate(options.producerLeaseUntil, 'producerLeaseUntil');
+ if (producerLeaseUntil != null && options.requiredWorkerCapability == null) {
+ throw new AgentTriggerDeliveryError('producerLeaseUntil requires a capability-fenced delivery');
+ }
if (
envelope.expectedAction != null &&
(envelope.mode !== 'continue' ||
@@ -195,6 +212,7 @@ export function prepareAgentTriggerDelivery(
...(options.requiredWorkerCapability == null
? {}
: { requiredWorkerCapability: options.requiredWorkerCapability }),
+ ...(producerLeaseUntil == null ? {} : { producerLeaseUntil }),
...(coalesceKey != null &&
coalesceUntil != null && {
coalesceKey,
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index de458161e3..bdc827c2f4 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -1,4 +1,5 @@
import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
getTenantId,
SYSTEM_TENANT_ID,
@@ -99,6 +100,8 @@ function deliveryMethods(
beginAgentTriggerDeliveryAttempt: jest.fn(async () => 1),
deferAgentTriggerDeliveryAttempt: jest.fn(async () => true),
completeAgentTriggerDelivery: jest.fn(async () => true),
+ retireAgentTriggerDelivery: jest.fn(async () => true),
+ renewAgentTriggerDeliveryProducerLease: jest.fn(async () => true),
retryAgentTriggerDelivery: jest.fn(async () => true),
deadLetterAgentTriggerDelivery: jest.fn(async () => true),
getAgentTriggerDelivery: jest.fn(async () => null),
@@ -175,7 +178,10 @@ describe('durable agent trigger service', () => {
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
expect.objectContaining({
- workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],
+ workerCapabilities: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ ],
}),
);
expect(methods.enqueueAgentTriggerDelivery).toHaveBeenCalledWith(
@@ -197,7 +203,7 @@ describe('durable agent trigger service', () => {
await service.stop();
});
- it('does not advertise detached completion capability without durable generation storage', async () => {
+ it('advertises ordinary completion but not detached-action capability without durable storage', async () => {
const methods = deliveryMethods();
const service = createAgentTriggerService({
methods,
@@ -208,7 +214,9 @@ describe('durable agent trigger service', () => {
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
- expect.objectContaining({ workerCapabilities: [] }),
+ expect.objectContaining({
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1],
+ }),
);
await service.stop();
});
@@ -418,9 +426,14 @@ describe('durable agent trigger service', () => {
expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
return deliveryRecord();
});
+ const retireAgentTriggerDelivery = jest.fn(async () => {
+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
+ return true;
+ });
const methods = deliveryMethods({
getAgentTriggerDeadLetters,
requeueAgentTriggerDelivery,
+ retireAgentTriggerDelivery,
});
const service = createAgentTriggerService({
methods,
@@ -434,6 +447,27 @@ describe('durable agent trigger service', () => {
await expect(service.requeue('delivery-row-1', START)).resolves.toMatchObject({
status: 'pending',
});
+ await expect(
+ service.retire('trigger_1', 'background-tool-completion', 'result unavailable', {
+ onlyIfUnclaimed: true,
+ }),
+ ).resolves.toBe(true);
+ expect(retireAgentTriggerDelivery).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'trigger_1',
+ sourceId: 'background-tool-completion',
+ reason: 'result unavailable',
+ onlyIfUnclaimed: true,
+ }),
+ );
+ await expect(
+ service.retire('trigger_1', 'background-tool-completion', 'dead recovery', {
+ onlyIfDead: true,
+ }),
+ ).resolves.toBe(true);
+ expect(retireAgentTriggerDelivery).toHaveBeenLastCalledWith(
+ expect.objectContaining({ onlyIfDead: true }),
+ );
await service.stop();
});
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index 554471c291..b8f51f0274 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -1,9 +1,13 @@
import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
logger,
runAsSystem,
} from '@librechat/data-schemas';
-import type { AgentTriggerDeliveryStatusRecord } from '@librechat/data-schemas';
+import type {
+ AgentTriggerDeliveryMethods,
+ AgentTriggerDeliveryStatusRecord,
+} from '@librechat/data-schemas';
import type {
AgentTriggerDeliveryFailure,
AgentTriggerDeliveryEngine,
@@ -102,6 +106,8 @@ export interface AgentTriggerDeliveryPersistence {
beginAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['beginAttempt'];
deferAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['defer'];
completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];
+ retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];
+ renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];
retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
@@ -151,6 +157,13 @@ export interface AgentTriggerService {
) => Promise<AgentTriggerDeliveryStatusRecord | null>;
getDeadLetters: (limit?: number) => Promise<AgentTriggerStoredRecord[]>;
requeue: (id: string, availableAt?: Date) => Promise<AgentTriggerStoredRecord | null>;
+ retire: (
+ deliveryKey: string,
+ sourceId: string,
+ reason: string,
+ options?: { onlyIfUnclaimed?: boolean; onlyIfDead?: boolean },
+ ) => Promise<boolean>;
+ renewProducerLease: (deliveryKey: string, sourceId: string, leaseUntil: Date) => Promise<boolean>;
drainUser: (userId: string) => Promise<void>;
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
@@ -165,9 +178,12 @@ function createDeliveryStore(
claimNext: (input) =>
methods.claimNextAgentTriggerDelivery({
...input,
- workerCapabilities: supportsDetachedActionCompletion()
- ? [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1]
- : [],
+ workerCapabilities: [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ ...(supportsDetachedActionCompletion()
+ ? [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1]
+ : []),
+ ],
}),
findEarlierUnsettled: methods.findEarlierAgentTriggerDelivery,
getBatch: methods.getAgentTriggerDeliveryBatch,
@@ -504,6 +520,29 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
return revived;
}),
+ retire: (deliveryKey, sourceId, reason, options) =>
+ runAsSystem(async () => {
+ const retired = await requireCleanupMethods().retireAgentTriggerDelivery({
+ deliveryKey,
+ sourceId,
+ reason,
+ settledAt: new Date(),
+ ...(options?.onlyIfUnclaimed === true ? { onlyIfUnclaimed: true } : {}),
+ ...(options?.onlyIfDead === true ? { onlyIfDead: true } : {}),
+ });
+ if (retired) {
+ deliveryEngine?.wake();
+ }
+ return retired;
+ }),
+ renewProducerLease: (deliveryKey, sourceId, leaseUntil) =>
+ runAsSystem(async () =>
+ requireMethods().renewAgentTriggerDeliveryProducerLease({
+ deliveryKey,
+ sourceId,
+ leaseUntil,
+ }),
+ ),
drainUser,
prepareUserPurge: (userId, fenceStartedAt, tenantId) =>
runAsSystem(async () =>
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 5eca68126f..bf1c4c0eca 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -2939,6 +2939,55 @@ class GenerationJobManagerClass {
);
}
+ /** Atomically prevents an unpublished automatic continuation from starting
+ * after its delivery has been retired for manual recovery. If recovery wins
+ * the unstarted idempotency claim (directly or through the existing takeover
+ * CAS), it converts that exact token into a started tombstone. A concurrent
+ * `createJob` still holding the predecessor token then fails its atomic
+ * create, while later duplicate POSTs take the settled/refetch path.
+ *
+ * `started` means job creation won the claim race; the caller must inspect
+ * the generation itself before releasing any result ownership. */
+ async fenceGenerationClaimForRecovery(
+ userId: string,
+ clientRequestId: string,
+ streamId: string,
+ conversationId: string,
+ ): Promise<'fenced' | 'started' | 'unavailable'> {
+ const observed = await this.claimGeneration(userId, clientRequestId, streamId, conversationId);
+ if (observed.existing == null) {
+ return 'unavailable';
+ }
+ if (observed.existing.startedAt != null) {
+ return 'started';
+ }
+
+ let owned = observed;
+ if (!observed.claimed) {
+ owned = await this.takeoverGeneration(userId, clientRequestId, streamId, observed.existing);
+ if (owned.existing?.startedAt != null) {
+ return 'started';
+ }
+ if (!owned.claimed || owned.existing == null) {
+ return 'unavailable';
+ }
+ }
+
+ const claim = normalizeTokenClaim(owned.existing, 'background completion recovery fence');
+ if (claim.startedAt != null) {
+ return 'started';
+ }
+ await this.tombstoneObservedGenerationClaim(
+ userId,
+ clientRequestId,
+ streamId,
+ claim,
+ Date.now(),
+ claim.generationProtocolVersion === 2 ? 2 : 1,
+ );
+ return 'fenced';
+ }
+
private generationClaimKey(userId: string, clientRequestId: string, streamId: string): string {
return `{${streamId}}:${userId}:${clientRequestId}`;
}
diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
index f1b0257451..8427eb4585 100644
--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
@@ -341,6 +341,75 @@ describe('GenerationJobManager start-generation claim', () => {
}
});
+ it('fences a missing continuation claim as a settled recovery tombstone', async () => {
+ await expect(
+ manager.fenceGenerationClaimForRecovery(
+ 'user-1',
+ 'req-recovered',
+ 'stream-recovered',
+ 'stream-recovered',
+ ),
+ ).resolves.toBe('fenced');
+
+ await expect(
+ manager.claimGeneration('user-1', 'req-recovered', 'stream-recovered', 'stream-recovered'),
+ ).resolves.toMatchObject({
+ claimed: false,
+ existing: { startedAt: expect.any(Number) },
+ });
+ });
+
+ it('invalidates an unpublished continuation creator before manual recovery', async () => {
+ const original = await manager.claimGeneration(
+ 'user-1',
+ 'req-recovery-race',
+ 'stream-recovery-race',
+ 'stream-recovery-race',
+ 2,
+ );
+
+ await expect(
+ manager.fenceGenerationClaimForRecovery(
+ 'user-1',
+ 'req-recovery-race',
+ 'stream-recovery-race',
+ 'stream-recovery-race',
+ ),
+ ).resolves.toBe('fenced');
+ await expect(
+ manager.createJob('stream-recovery-race', 'user-1', 'stream-recovery-race', {
+ idempotencyClientRequestId: 'req-recovery-race',
+ idempotencyClaimToken: original.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('Generation idempotency claim was taken over before job creation');
+ await expect(store.getJob('stream-recovery-race')).resolves.toBeNull();
+ });
+
+ it('reports started when generation creation wins the recovery fence', async () => {
+ const claim = await manager.claimGeneration(
+ 'user-1',
+ 'req-created-first',
+ 'stream-created-first',
+ 'stream-created-first',
+ 2,
+ );
+ await manager.createJob('stream-created-first', 'user-1', 'stream-created-first', {
+ idempotencyClientRequestId: 'req-created-first',
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ await expect(
+ manager.fenceGenerationClaimForRecovery(
+ 'user-1',
+ 'req-created-first',
+ 'stream-created-first',
+ 'stream-created-first',
+ ),
+ ).resolves.toBe('started');
+ });
+
it('accepts an exact legacy started mark whose committed reply was lost', async () => {
const streamId = 'stream-lost-legacy-mark-reply';
const clientRequestId = 'req-lost-legacy-mark-reply';
diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts
index f86fb37f65..b20c691fa0 100644
--- a/packages/data-provider/src/schemas.ts
+++ b/packages/data-provider/src/schemas.ts
@@ -957,6 +957,10 @@ export type TAttachmentMetadata = {
type?: Tools;
messageId: string;
toolCallId: string;
+ /** Saved-agent owner when provider tool-call ids repeat across handoffs. */
+ agentId?: string;
+ /** Host run-step owner when one agent repeats a provider tool-call id. */
+ stepId?: string;
workspaceChange?: WorkspaceChange;
[Tools.memory]?: MemoryArtifact;
[Tools.ui_resources]?: UIResource[];
diff --git a/packages/data-provider/src/types/agents.ts b/packages/data-provider/src/types/agents.ts
index a51d2870d9..867cd7c9c6 100644
--- a/packages/data-provider/src/types/agents.ts
+++ b/packages/data-provider/src/types/agents.ts
@@ -79,8 +79,25 @@ export namespace Agents {
/** If provided, an identifier associated with the tool call */
id?: string;
+ /** Host run-step identity; unlike provider ids, this is unique across turns. */
+ stepId?: string;
/** If provided, the output of the tool call */
output?: string;
+ /** Host-owned durable receipt for a detached ordinary tool result.
+ * It lives beside the original call so reload, manual collection, and
+ * automatic continuation all arbitrate the same result identity. */
+ backgroundTask?: {
+ version: 1;
+ taskId: string;
+ toolName: string;
+ status: 'completed' | 'error';
+ settledAt: Date;
+ resultClaim?: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ claimedAt: Date;
+ };
+ };
/** The tool call was rejected before execution because its input failed schema validation. */
inputValidationError?: true;
/** Auth URL */
diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts
index 2083f24f09..2a581dce95 100644
--- a/packages/data-schemas/src/index.ts
+++ b/packages/data-schemas/src/index.ts
@@ -61,7 +61,10 @@ export {
MAX_COMPACTION_SEMANTIC_INDEX_TEXT_LENGTH,
isCompactionSemanticIndexProjection,
} from './types/compaction';
-export { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from './types/triggerDelivery';
+export {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+} from './types/triggerDelivery';
export type * from './types';
export type * from './methods';
export {
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index dbcd724fa8..65fb1a1a7d 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -65,6 +65,8 @@ import {
type ParentSubagentTaskRecord,
type SubagentThreadViewMessageRecord,
type SubagentTaskResultClaim,
+ type BackgroundToolResultClaim,
+ type BackgroundToolResultRecord,
} from './message';
import {
createConversationMethods,
@@ -126,6 +128,7 @@ import {
recordAgentEventActorReceiptMetric,
setAgentEventActorReceiptMetricObserver,
type AgentTriggerDeliveryMethods,
+ type AgentTriggerProducerLeaseStatus,
type AgentEventActorReceiptMetric,
type AgentEventActorReceiptStorageMetrics,
} from './triggerDelivery';
@@ -424,6 +427,8 @@ export type {
ParentSubagentThreadRecord,
SubagentThreadViewMessageRecord,
SubagentTaskResultClaim,
+ BackgroundToolResultClaim,
+ BackgroundToolResultRecord,
ConversationMethods,
AgentEventActorReconciliationStorageMetrics,
ChatProjectMethods,
@@ -446,6 +451,7 @@ export type {
UpsertSkillSyncCredentialInput,
SkillSyncMethods,
AgentTriggerDeliveryMethods,
+ AgentTriggerProducerLeaseStatus,
AgentEventActorReceiptMetric,
AgentEventActorReceiptStorageMetrics,
ScheduleMethods,
diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts
index cbb21ea343..28de4cc2f6 100644
--- a/packages/data-schemas/src/methods/message.spec.ts
+++ b/packages/data-schemas/src/methods/message.spec.ts
@@ -53,6 +53,12 @@ let getSubagentTaskControlReplay: ReturnType<
let releaseSubagentTaskResultClaim: ReturnType<
typeof createMessageMethods
>['releaseSubagentTaskResultClaim'];
+let claimBackgroundToolResults: ReturnType<
+ typeof createMessageMethods
+>['claimBackgroundToolResults'];
+let releaseBackgroundToolResultClaims: ReturnType<
+ typeof createMessageMethods
+>['releaseBackgroundToolResultClaims'];
function rejectUpdateArrays(beforeUpdate?: () => Promise<void>) {
const originalFindOneAndUpdate = Message.collection.findOneAndUpdate.bind(Message.collection);
@@ -90,6 +96,8 @@ beforeAll(async () => {
getSubagentTaskControlReceipt = methods.getSubagentTaskControlReceipt;
getSubagentTaskControlReplay = methods.getSubagentTaskControlReplay;
releaseSubagentTaskResultClaim = methods.releaseSubagentTaskResultClaim;
+ claimBackgroundToolResults = methods.claimBackgroundToolResults;
+ releaseBackgroundToolResultClaims = methods.releaseBackgroundToolResultClaims;
await mongoose.connect(mongoUri);
});
@@ -818,6 +826,67 @@ describe('Message Operations', () => {
expect(content[1].tool_call?.output).toBe('stdout-b');
});
+ it('patches only the matching run step when one agent repeats a provider id', async () => {
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [
+ {
+ type: 'tool_call',
+ agentId: 'agent_a',
+ tool_call: { id: 'call_0', stepId: 'step-1', name: 'slow_tool', output: 'handle-1' },
+ },
+ {
+ type: 'tool_call',
+ agentId: 'agent_a',
+ tool_call: { id: 'call_0', stepId: 'step-2', name: 'slow_tool', output: 'handle-2' },
+ },
+ ],
+ });
+
+ const result = await updateToolCallResult({
+ userId: 'user123',
+ messageId: 'msg123',
+ conversationId: mockMessageData.conversationId as string,
+ toolCallId: 'call_0',
+ stepId: 'step-2',
+ agentId: 'agent_a',
+ output: 'terminal-2',
+ backgroundTask: {
+ taskId: 'task-2',
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ },
+ });
+ expect(result).toEqual({ matched: true, unfinished: false });
+
+ const saved = await Message.findOne({ messageId: 'msg123', user: 'user123' }).lean();
+ const content = saved?.content as Array<{
+ tool_call?: { output?: string; backgroundTask?: { taskId?: string } };
+ }>;
+ expect(content[0].tool_call).toMatchObject({ output: 'handle-1' });
+ expect(content[0].tool_call?.backgroundTask).toBeUndefined();
+ expect(content[1].tool_call).toMatchObject({
+ output: 'terminal-2',
+ backgroundTask: { taskId: 'task-2' },
+ });
+ });
+
+ it('reports no match when the targeted tool-call part is absent', async () => {
+ await saveMessage(mockCtx, { ...mockMessageData, content: toolCallContent() });
+
+ await expect(
+ updateToolCallResult({
+ userId: 'user123',
+ messageId: 'msg123',
+ conversationId: mockMessageData.conversationId as string,
+ toolCallId: 'call_bg',
+ stepId: 'missing-step',
+ output: 'must not persist',
+ }),
+ ).resolves.toEqual({ matched: false, unfinished: false });
+ });
+
it('flags unfinished partial rows so callers keep re-applying until finalize', async () => {
await saveMessage(mockCtx, {
...mockMessageData,
@@ -837,6 +906,403 @@ describe('Message Operations', () => {
expect(result).toEqual({ matched: true, unfinished: true });
});
+ it('persists a terminal background receipt without replacing its existing claim', async () => {
+ const content = toolCallContent();
+ (content[1].tool_call as Record<string, unknown>).backgroundTask = {
+ version: 1,
+ taskId: 'task-1',
+ toolName: 'execute_code',
+ status: 'completed',
+ settledAt: new Date('2026-08-30T12:00:00Z'),
+ resultClaim: {
+ kind: 'manual',
+ claimId: 'manual-1',
+ claimedAt: new Date('2026-08-30T12:00:01Z'),
+ },
+ };
+ await saveMessage(mockCtx, { ...mockMessageData, content });
+
+ await updateToolCallResult({
+ userId: 'user123',
+ messageId: 'msg123',
+ conversationId: mockMessageData.conversationId as string,
+ toolCallId: 'call_bg',
+ output: 'terminal output',
+ backgroundTask: {
+ taskId: 'task-1',
+ toolName: 'execute_code',
+ status: 'completed',
+ settledAt: new Date('2026-08-30T12:00:02Z'),
+ resultClaim: {
+ kind: 'wakeup',
+ claimId: 'wakeup-1',
+ claimedAt: new Date('2026-08-30T12:00:02Z'),
+ },
+ },
+ });
+
+ const saved = await Message.findOne({ messageId: 'msg123', user: 'user123' }).lean();
+ const task = (
+ saved?.content?.[1] as {
+ tool_call?: { backgroundTask?: Record<string, unknown> };
+ }
+ ).tool_call?.backgroundTask;
+ expect(task).toMatchObject({
+ version: 1,
+ taskId: 'task-1',
+ status: 'completed',
+ resultClaim: { kind: 'manual', claimId: 'manual-1' },
+ });
+ });
+
+ it('elects one result consumer and batches terminal siblings for a wakeup', async () => {
+ const terminal = (id: string, taskId: string, output: string) => ({
+ type: 'tool_call',
+ agentId: 'agent-a',
+ tool_call: {
+ id,
+ name: 'slow_tool',
+ output,
+ backgroundTask: {
+ version: 1,
+ taskId,
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ },
+ },
+ });
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [terminal('call-1', 'task-1', 'one'), terminal('call-2', 'task-2', 'two')],
+ });
+
+ const wakeup = await claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-1',
+ kind: 'wakeup',
+ claimId: 'delivery-1',
+ });
+ expect(wakeup).toEqual({
+ status: 'acquired',
+ results: [
+ {
+ taskId: 'task-1',
+ toolCallId: 'call-1',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'one',
+ agentId: 'agent-a',
+ },
+ {
+ taskId: 'task-2',
+ toolCallId: 'call-2',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'two',
+ agentId: 'agent-a',
+ },
+ ],
+ });
+ await Message.updateOne(
+ { messageId: 'msg123', user: 'user123' },
+ { $push: { content: terminal('call-3', 'task-3', 'three') } },
+ );
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-1',
+ kind: 'wakeup',
+ claimId: 'delivery-1',
+ }),
+ ).resolves.toEqual(wakeup);
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-2',
+ kind: 'manual',
+ claimId: 'poll-1',
+ }),
+ ).resolves.toEqual({
+ status: 'claimed',
+ claim: { kind: 'wakeup', claimId: 'delivery-1' },
+ });
+ });
+
+ it('revalidates each sibling inside the atomic batch claim', async () => {
+ const terminal = (taskId: string) => ({
+ type: 'tool_call',
+ tool_call: {
+ id: `call-${taskId}`,
+ name: 'slow_tool',
+ output: taskId,
+ backgroundTask: {
+ version: 1,
+ taskId,
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ },
+ },
+ });
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [terminal('root'), terminal('stale-sibling')],
+ });
+ const originalFindOneAndUpdate = Message.collection.findOneAndUpdate.bind(Message.collection);
+ const claimWrite = jest
+ .spyOn(Message.collection, 'findOneAndUpdate')
+ .mockImplementationOnce(async (...args) => {
+ await Message.collection.updateOne(
+ { user: 'user123', messageId: 'msg123' },
+ { $set: { 'content.1.tool_call.backgroundTask.status': 'running' } },
+ );
+ return originalFindOneAndUpdate(...args);
+ });
+
+ try {
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'root',
+ kind: 'wakeup',
+ claimId: 'delivery-root',
+ }),
+ ).resolves.toMatchObject({
+ status: 'acquired',
+ results: [expect.objectContaining({ taskId: 'root' })],
+ });
+ } finally {
+ claimWrite.mockRestore();
+ }
+
+ const saved = await Message.findOne({ user: 'user123', messageId: 'msg123' }).lean();
+ expect(saved?.content?.[1]).not.toHaveProperty('tool_call.backgroundTask.resultClaim');
+ });
+
+ it('does not batch a terminal sibling that elected poll-only delivery', async () => {
+ const terminal = (taskId: string, completionWakeup: boolean) => ({
+ type: 'tool_call',
+ tool_call: {
+ id: `call-${taskId}`,
+ name: 'slow_tool',
+ output: taskId,
+ backgroundTask: {
+ version: 1,
+ taskId,
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ ...(completionWakeup ? { completionWakeup: true } : {}),
+ },
+ },
+ });
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [terminal('wakeup', true), terminal('poll-only', false)],
+ });
+
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'wakeup',
+ kind: 'wakeup',
+ claimId: 'delivery-1',
+ }),
+ ).resolves.toMatchObject({
+ status: 'acquired',
+ results: [expect.objectContaining({ taskId: 'wakeup' })],
+ });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'poll-only',
+ kind: 'manual',
+ claimId: 'poll-1',
+ }),
+ ).resolves.toMatchObject({
+ status: 'acquired',
+ results: [expect.objectContaining({ taskId: 'poll-only' })],
+ });
+ });
+
+ it('uses nested tool-call agent identity when batching sibling results', async () => {
+ const backgroundTask = (taskId: string) => ({
+ version: 1,
+ taskId,
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ });
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [
+ {
+ type: 'tool_call',
+ agentId: 'agent-a',
+ tool_call: {
+ id: 'call-a',
+ name: 'slow_tool',
+ output: 'a',
+ backgroundTask: backgroundTask('task-a'),
+ },
+ },
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-b',
+ agentId: 'agent-b',
+ name: 'slow_tool',
+ output: 'b',
+ backgroundTask: backgroundTask('task-b'),
+ },
+ },
+ ],
+ });
+
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-a',
+ agentId: 'agent-a',
+ kind: 'wakeup',
+ claimId: 'delivery-a',
+ }),
+ ).resolves.toEqual({
+ status: 'acquired',
+ results: [
+ {
+ taskId: 'task-a',
+ toolCallId: 'call-a',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'a',
+ agentId: 'agent-a',
+ },
+ ],
+ });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-b',
+ agentId: 'agent-b',
+ kind: 'manual',
+ claimId: 'poll-b',
+ }),
+ ).resolves.toMatchObject({
+ status: 'acquired',
+ results: [expect.objectContaining({ taskId: 'task-b', agentId: 'agent-b' })],
+ });
+ });
+
+ it('releases only the exact wakeup batch claim', async () => {
+ const task = {
+ version: 1,
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ resultClaim: { kind: 'wakeup', claimId: 'delivery-1', claimedAt: new Date() },
+ };
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-1',
+ name: 'slow_tool',
+ output: 'one',
+ backgroundTask: { ...task, taskId: 'task-1' },
+ },
+ },
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-2',
+ name: 'slow_tool',
+ output: 'two',
+ backgroundTask: {
+ ...task,
+ taskId: 'task-2',
+ resultClaim: { kind: 'manual', claimId: 'poll-1', claimedAt: new Date() },
+ },
+ },
+ },
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-3',
+ name: 'slow_tool',
+ output: 'three',
+ backgroundTask: { ...task, taskId: 'task-3' },
+ },
+ },
+ ],
+ });
+
+ await expect(
+ releaseBackgroundToolResultClaims({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ kind: 'wakeup',
+ claimId: 'delivery-1',
+ }),
+ ).resolves.toBe(true);
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-1',
+ kind: 'manual',
+ claimId: 'poll-2',
+ }),
+ ).resolves.toMatchObject({ status: 'acquired' });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-3',
+ kind: 'manual',
+ claimId: 'poll-3',
+ }),
+ ).resolves.toMatchObject({ status: 'acquired' });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-2',
+ kind: 'wakeup',
+ claimId: 'delivery-2',
+ }),
+ ).resolves.toEqual({
+ status: 'claimed',
+ claim: { kind: 'manual', claimId: 'poll-1' },
+ });
+ });
+
it('keeps a sibling AGENT’s attachment when both id and file key collide', async () => {
/* Handoff agents can share a provider tool-call id AND a claimed
* file_id (same filename in one conversation); the second agent's
@@ -974,7 +1440,27 @@ describe('Message Operations', () => {
isCreatedByUser: false,
sender: 'Agent',
text: 'visible text',
- content: [{ type: 'text', text: 'part text' }],
+ content: [
+ { type: 'text', text: 'part text' },
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-background',
+ backgroundTask: {
+ version: 1,
+ taskId: 'task-visible',
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ resultClaim: {
+ kind: 'wakeup',
+ claimId: 'private-delivery-identity',
+ claimedAt: new Date(),
+ },
+ },
+ },
+ },
+ ],
tokenCount: 42,
conversationSignature: 'sig',
clientId: 'client-1',
@@ -1030,7 +1516,12 @@ describe('Message Operations', () => {
);
expect(message.text).toBe('visible text');
- expect(message.content).toHaveLength(1);
+ expect(message.content).toHaveLength(2);
+ const projectedTask = (
+ message.content?.[1] as { tool_call?: { backgroundTask?: Record<string, unknown> } }
+ ).tool_call?.backgroundTask;
+ expect(projectedTask).toMatchObject({ taskId: 'task-visible' });
+ expect(projectedTask).not.toHaveProperty('resultClaim');
expect(message.tokenCount).toBe(42);
const metadata = message.metadata as Record<string, unknown>;
expect(metadata.usage).toBeDefined();
diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts
index e6b0bd786f..5d2729de60 100644
--- a/packages/data-schemas/src/methods/message.ts
+++ b/packages/data-schemas/src/methods/message.ts
@@ -380,6 +380,8 @@ export const CLIENT_MESSAGE_SELECT: string = [
'-langfuseSampled',
'-langfuseDestinationIds',
'-metadata.thoughtSignatures',
+ '-content.tool_call.backgroundTask.resultClaim',
+ '-content.tool_call.backgroundTask.completionWakeup',
'-attachments.web_search.knowledgeGraph',
'-attachments.web_search.peopleAlsoAsk',
'-attachments.web_search.relatedSearches',
@@ -401,6 +403,20 @@ export type SubagentTaskResultClaim =
| { status: 'claimed'; message: IMessage }
| { status: 'acquired'; message: IMessage };
+export interface BackgroundToolResultRecord {
+ taskId: string;
+ toolCallId: string;
+ toolName: string;
+ status: 'completed' | 'error';
+ output: string;
+ agentId?: string;
+}
+
+export type BackgroundToolResultClaim =
+ | { status: 'not_found' | 'not_ready' }
+ | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }
+ | { status: 'acquired'; results: BackgroundToolResultRecord[] };
+
export type SubagentThreadViewMessageRecord = Pick<
IMessage,
| 'messageId'
@@ -507,11 +523,43 @@ export interface MessageMethods {
messageId: string;
conversationId: string;
toolCallId: string;
+ stepId?: string;
agentId?: string;
output?: string;
attachments?: unknown[];
markBackgrounded?: boolean;
+ backgroundTask?: {
+ taskId: string;
+ toolName: string;
+ status: 'completed' | 'error';
+ settledAt: Date;
+ completionWakeup?: true;
+ resultClaim?: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ claimedAt: Date;
+ };
+ };
}): Promise<{ matched: boolean; unfinished: boolean }>;
+ claimBackgroundToolResults(params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskId: string;
+ agentId?: string;
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ limit?: number;
+ }): Promise<BackgroundToolResultClaim>;
+ releaseBackgroundToolResultClaims(params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ /** Omit to release every sibling owned by this exact batch claim. */
+ taskIds?: string[];
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ }): Promise<boolean>;
updateMessage(
userId: string,
message: Partial<IMessage> & { newMessageId?: string },
@@ -872,15 +920,18 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
messageId,
conversationId,
toolCallId,
+ stepId,
agentId,
output,
attachments,
markBackgrounded,
+ backgroundTask,
}: {
userId: string;
messageId: string;
conversationId: string;
toolCallId: string;
+ stepId?: string;
/** Scopes the part match when provider tool-call ids repeat across
* agents in one response message (e.g. `call_0` per response); a part
* without agent identity matches any caller (single-agent runs). */
@@ -895,9 +946,40 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
* that erases it must persist a durable one alongside.
*/
markBackgrounded?: boolean;
+ backgroundTask?: {
+ taskId: string;
+ toolName: string;
+ status: 'completed' | 'error';
+ settledAt: Date;
+ completionWakeup?: true;
+ resultClaim?: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ claimedAt: Date;
+ };
+ };
}): Promise<{ matched: boolean; unfinished: boolean }> {
+ const targetPartMatch = {
+ $and: [
+ { $eq: ['$$part.type', 'tool_call'] },
+ { $eq: ['$$part.tool_call.id', toolCallId] },
+ ...(stepId != null ? [{ $eq: ['$$part.tool_call.stepId', stepId] }] : []),
+ ...(agentId != null
+ ? [
+ {
+ $in: [
+ {
+ $ifNull: [{ $ifNull: ['$$part.agentId', '$$part.tool_call.agentId'] }, null],
+ },
+ [null, agentId],
+ ],
+ },
+ ]
+ : []),
+ ],
+ };
const stages: Record<string, unknown>[] = [];
- if (output !== undefined) {
+ if (output !== undefined || backgroundTask != null) {
stages.push({
$set: {
content: {
@@ -906,22 +988,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
as: 'part',
in: {
$cond: [
- {
- $and: [
- { $eq: ['$$part.type', 'tool_call'] },
- { $eq: ['$$part.tool_call.id', toolCallId] },
- ...(agentId != null
- ? [
- {
- $in: [
- { $ifNull: ['$$part.agentId', '$$part.tool_call.agentId'] },
- [null, agentId],
- ],
- },
- ]
- : []),
- ],
- },
+ targetPartMatch,
{
$mergeObjects: [
'$$part',
@@ -930,8 +997,54 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
$mergeObjects: [
'$$part.tool_call',
{
- output: { $literal: output },
+ ...(output !== undefined ? { output: { $literal: output } } : {}),
...(markBackgrounded === true ? { backgrounded: true } : {}),
+ ...(backgroundTask != null
+ ? {
+ backgroundTask: {
+ $mergeObjects: [
+ {
+ $literal: {
+ version: 1,
+ taskId: backgroundTask.taskId,
+ toolName: backgroundTask.toolName,
+ status: backgroundTask.status,
+ settledAt: backgroundTask.settledAt,
+ ...(backgroundTask.completionWakeup === true
+ ? { completionWakeup: true }
+ : {}),
+ },
+ },
+ {
+ $cond: [
+ {
+ $ne: [
+ {
+ $ifNull: [
+ '$$part.tool_call.backgroundTask.resultClaim',
+ null,
+ ],
+ },
+ null,
+ ],
+ },
+ {
+ resultClaim:
+ '$$part.tool_call.backgroundTask.resultClaim',
+ },
+ backgroundTask.resultClaim != null
+ ? {
+ $literal: {
+ resultClaim: backgroundTask.resultClaim,
+ },
+ }
+ : {},
+ ],
+ },
+ ],
+ },
+ }
+ : {}),
},
],
},
@@ -990,6 +1103,13 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
},
]
: []),
+ ...(stepId != null
+ ? [
+ {
+ $in: [{ $ifNull: ['$$existing.stepId', null] }, [null, stepId]],
+ },
+ ]
+ : []),
],
},
],
@@ -1008,7 +1128,20 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
try {
const Message = mongoose.models.Message as Model<IMessage>;
const result = await Message.findOneAndUpdate(
- { messageId, user: userId, conversationId },
+ {
+ messageId,
+ user: userId,
+ conversationId,
+ $expr: {
+ $anyElementTrue: {
+ $map: {
+ input: { $ifNull: ['$content', []] },
+ as: 'part',
+ in: targetPartMatch,
+ },
+ },
+ },
+ },
stages,
{ new: true, projection: { unfinished: 1 } },
).lean<{ unfinished?: boolean } | null>();
@@ -1019,6 +1152,438 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
}
}
+ const MAX_BACKGROUND_TOOL_RESULT_BATCH = 8;
+
+ function readBackgroundToolResultClaim(
+ row: Pick<IMessage, 'content'>,
+ taskId: string,
+ ): { kind: 'manual' | 'wakeup'; claimId: string } | undefined {
+ for (const part of row.content ?? []) {
+ if (part == null || typeof part !== 'object' || Array.isArray(part)) {
+ continue;
+ }
+ const backgroundTask = (
+ part as {
+ tool_call?: {
+ backgroundTask?: {
+ taskId?: unknown;
+ resultClaim?: { kind?: unknown; claimId?: unknown };
+ };
+ };
+ }
+ ).tool_call?.backgroundTask;
+ if (backgroundTask?.taskId !== taskId) {
+ continue;
+ }
+ const claim = backgroundTask.resultClaim;
+ if (
+ (claim?.kind === 'manual' || claim?.kind === 'wakeup') &&
+ typeof claim.claimId === 'string' &&
+ claim.claimId.length > 0
+ ) {
+ return { kind: claim.kind, claimId: claim.claimId };
+ }
+ return;
+ }
+ }
+
+ function parseBackgroundToolResults(
+ message: IMessage,
+ claim: { kind: 'manual' | 'wakeup'; claimId: string },
+ ): BackgroundToolResultRecord[] {
+ const results: BackgroundToolResultRecord[] = [];
+ for (const part of message.content ?? []) {
+ if (part == null || typeof part !== 'object' || Array.isArray(part)) {
+ continue;
+ }
+ const record = part as {
+ agentId?: unknown;
+ tool_call?: {
+ id?: unknown;
+ agentId?: unknown;
+ output?: unknown;
+ backgroundTask?: {
+ taskId?: unknown;
+ toolName?: unknown;
+ status?: unknown;
+ resultClaim?: { kind?: unknown; claimId?: unknown };
+ };
+ };
+ };
+ const toolCall = record.tool_call;
+ const task = toolCall?.backgroundTask;
+ if (
+ typeof toolCall?.id !== 'string' ||
+ typeof task?.taskId !== 'string' ||
+ typeof task.toolName !== 'string' ||
+ (task.status !== 'completed' && task.status !== 'error') ||
+ task.resultClaim?.kind !== claim.kind ||
+ task.resultClaim.claimId !== claim.claimId
+ ) {
+ continue;
+ }
+ let resultAgentId: string | undefined;
+ if (typeof record.agentId === 'string') {
+ resultAgentId = record.agentId;
+ } else if (typeof toolCall.agentId === 'string') {
+ resultAgentId = toolCall.agentId;
+ }
+ results.push({
+ taskId: task.taskId,
+ toolCallId: toolCall.id,
+ toolName: task.toolName,
+ status: task.status,
+ output: typeof toolCall.output === 'string' ? toolCall.output : '',
+ ...(resultAgentId == null ? {} : { agentId: resultAgentId }),
+ });
+ }
+ return results;
+ }
+
+ /** Atomically elects manual polling or one automatic continuation. Wakeups
+ * also claim a bounded set of already-settled siblings from the same parent
+ * response, avoiding one paid continuation per concurrently completed tool. */
+ async function claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId,
+ taskId,
+ agentId,
+ kind,
+ claimId,
+ limit = kind === 'wakeup' ? MAX_BACKGROUND_TOOL_RESULT_BATCH : 1,
+ }: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskId: string;
+ agentId?: string;
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ limit?: number;
+ }): Promise<BackgroundToolResultClaim> {
+ if (
+ messageId.length === 0 ||
+ messageId.length > 256 ||
+ taskId.length === 0 ||
+ taskId.length > 256 ||
+ claimId.length === 0 ||
+ claimId.length > 128 ||
+ (kind !== 'manual' && kind !== 'wakeup')
+ ) {
+ throw new TypeError('Invalid background tool result claim');
+ }
+ const boundedLimit = Math.max(1, Math.min(MAX_BACKGROUND_TOOL_RESULT_BATCH, limit));
+ const Message = mongoose.models.Message as Model<IMessage>;
+ const row = await Message.findOne({ user: userId, conversationId, messageId })
+ .select({ content: 1, unfinished: 1 })
+ .lean<IMessage | null>();
+ if (row == null) {
+ return { status: 'not_found' };
+ }
+ if (row.unfinished === true) {
+ return { status: 'not_ready' };
+ }
+ const requestedClaim = readBackgroundToolResultClaim(row, taskId);
+ const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;
+ const candidates: string[] = [];
+ let requestedState: 'ready' | 'claimed' | 'missing' = 'missing';
+ for (const part of row.content ?? []) {
+ if (part == null || typeof part !== 'object' || Array.isArray(part)) {
+ continue;
+ }
+ const task = (part as { tool_call?: { backgroundTask?: Record<string, unknown> } }).tool_call
+ ?.backgroundTask;
+ const partRecord = part as { agentId?: unknown; tool_call?: { agentId?: unknown } };
+ const partAgentId = partRecord.agentId ?? partRecord.tool_call?.agentId;
+ const candidateId = task?.taskId;
+ if (typeof candidateId !== 'string') {
+ continue;
+ }
+ const terminal = task?.status === 'completed' || task?.status === 'error';
+ const wakeupEligible = kind !== 'wakeup' || task?.completionWakeup === true;
+ const resultClaim = task?.resultClaim as { kind?: unknown; claimId?: unknown } | undefined;
+ const replay = resultClaim?.kind === kind && resultClaim.claimId === claimId;
+ const sameAgent =
+ agentId == null ||
+ partAgentId == null ||
+ (typeof partAgentId === 'string' && partAgentId === agentId);
+ /** A lost-receipt retry replays exactly its original assignment. It must
+ * not absorb siblings that completed after the already-admitted input
+ * was constructed, or those results would be claimed but never shown. */
+ const claimable =
+ terminal && wakeupEligible && sameAgent && (replaying ? replay : resultClaim == null);
+ if (candidateId === taskId) {
+ if (claimable) {
+ requestedState = 'ready';
+ } else if (resultClaim != null) {
+ requestedState = 'claimed';
+ }
+ }
+ if (
+ claimable &&
+ (candidateId === taskId || kind === 'wakeup') &&
+ candidates.length < boundedLimit
+ ) {
+ candidates.push(candidateId);
+ }
+ }
+ if (requestedState === 'missing') {
+ return { status: 'not_ready' };
+ }
+ if (requestedState === 'claimed') {
+ return {
+ status: 'claimed',
+ ...(requestedClaim == null ? {} : { claim: requestedClaim }),
+ };
+ }
+ if (!candidates.includes(taskId)) {
+ candidates.unshift(taskId);
+ candidates.splice(boundedLimit);
+ }
+ const claimedAt = new Date();
+ const updated = await Message.findOneAndUpdate(
+ {
+ user: userId,
+ conversationId,
+ messageId,
+ unfinished: { $ne: true },
+ content: {
+ $elemMatch: {
+ type: 'tool_call',
+ 'tool_call.backgroundTask.taskId': taskId,
+ 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
+ ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
+ ...(replaying
+ ? {
+ 'tool_call.backgroundTask.resultClaim.kind': kind,
+ 'tool_call.backgroundTask.resultClaim.claimId': claimId,
+ }
+ : { 'tool_call.backgroundTask.resultClaim': { $exists: false } }),
+ },
+ },
+ ...(agentId != null
+ ? {
+ $expr: {
+ $anyElementTrue: {
+ $map: {
+ input: { $ifNull: ['$content', []] },
+ as: 'candidate',
+ in: {
+ $and: [
+ { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
+ {
+ $in: [
+ {
+ $ifNull: [
+ {
+ $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
+ },
+ null,
+ ],
+ },
+ [null, agentId],
+ ],
+ },
+ ],
+ },
+ },
+ },
+ },
+ }
+ : {}),
+ },
+ [
+ {
+ $set: {
+ content: {
+ $map: {
+ input: { $ifNull: ['$content', []] },
+ as: 'part',
+ in: {
+ $cond: [
+ {
+ $and: [
+ { $eq: ['$$part.type', 'tool_call'] },
+ { $in: ['$$part.tool_call.backgroundTask.taskId', candidates] },
+ {
+ $in: ['$$part.tool_call.backgroundTask.status', ['completed', 'error']],
+ },
+ ...(kind === 'wakeup'
+ ? [{ $eq: ['$$part.tool_call.backgroundTask.completionWakeup', true] }]
+ : []),
+ ...(agentId != null
+ ? [
+ {
+ $in: [
+ {
+ $ifNull: [
+ { $ifNull: ['$$part.agentId', '$$part.tool_call.agentId'] },
+ null,
+ ],
+ },
+ [null, agentId],
+ ],
+ },
+ ]
+ : []),
+ {
+ $or: [
+ {
+ $eq: [
+ { $ifNull: ['$$part.tool_call.backgroundTask.resultClaim', null] },
+ null,
+ ],
+ },
+ {
+ $and: [
+ {
+ $eq: ['$$part.tool_call.backgroundTask.resultClaim.kind', kind],
+ },
+ {
+ $eq: [
+ '$$part.tool_call.backgroundTask.resultClaim.claimId',
+ claimId,
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ {
+ $mergeObjects: [
+ '$$part',
+ {
+ tool_call: {
+ $mergeObjects: [
+ '$$part.tool_call',
+ {
+ backgroundTask: {
+ $mergeObjects: [
+ '$$part.tool_call.backgroundTask',
+ {
+ resultClaim: {
+ kind,
+ claimId,
+ claimedAt,
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ '$$part',
+ ],
+ },
+ },
+ },
+ },
+ },
+ ],
+ { new: true, projection: { content: 1 } },
+ ).lean<IMessage | null>();
+ if (updated == null) {
+ return { status: 'not_ready' };
+ }
+ const results = parseBackgroundToolResults(updated, { kind, claimId });
+ const competingClaim = readBackgroundToolResultClaim(updated, taskId);
+ return results.some((result) => result.taskId === taskId)
+ ? { status: 'acquired', results }
+ : {
+ status: 'claimed',
+ ...(competingClaim == null ? {} : { claim: competingClaim }),
+ };
+ }
+
+ async function releaseBackgroundToolResultClaims({
+ userId,
+ conversationId,
+ messageId,
+ taskIds,
+ kind,
+ claimId,
+ }: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskIds?: string[];
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ }): Promise<boolean> {
+ if (taskIds?.length === 0) {
+ return true;
+ }
+ const Message = mongoose.models.Message as Model<IMessage>;
+ const updated = await Message.findOneAndUpdate(
+ { user: userId, conversationId, messageId },
+ [
+ {
+ $set: {
+ content: {
+ $map: {
+ input: { $ifNull: ['$content', []] },
+ as: 'part',
+ in: {
+ $cond: [
+ {
+ $and: [
+ ...(taskIds == null
+ ? []
+ : [{ $in: ['$$part.tool_call.backgroundTask.taskId', taskIds] }]),
+ { $eq: ['$$part.tool_call.backgroundTask.resultClaim.kind', kind] },
+ { $eq: ['$$part.tool_call.backgroundTask.resultClaim.claimId', claimId] },
+ ],
+ },
+ {
+ $mergeObjects: [
+ '$$part',
+ {
+ tool_call: {
+ $mergeObjects: [
+ '$$part.tool_call',
+ {
+ backgroundTask: {
+ $arrayToObject: {
+ $filter: {
+ input: {
+ $objectToArray: '$$part.tool_call.backgroundTask',
+ },
+ as: 'field',
+ cond: { $ne: ['$$field.k', 'resultClaim'] },
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ '$$part',
+ ],
+ },
+ },
+ },
+ },
+ },
+ ],
+ { new: true, projection: { content: 1 } },
+ ).lean<IMessage | null>();
+ if (updated == null) {
+ return false;
+ }
+ const remaining = parseBackgroundToolResults(updated, { kind, claimId });
+ return taskIds == null
+ ? remaining.length === 0
+ : !remaining.some((result) => taskIds.includes(result.taskId));
+ }
+
/**
* Updates a message and returns sanitized fields.
*/
@@ -2436,6 +3001,8 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
recordMessage,
updateMessageText,
updateToolCallResult,
+ claimBackgroundToolResults,
+ releaseBackgroundToolResultClaims,
updateMessage,
recordSubagentTaskControlReceipt,
getSubagentTaskControlReceipt,
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index c58a033878..d516f1c4f4 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -15,7 +15,10 @@ import {
setAgentEventActorReceiptMetricObserver,
type AgentTriggerDeliveryMethods,
} from './triggerDelivery';
-import { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from '~/types/triggerDelivery';
+import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+} from '~/types/triggerDelivery';
import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';
import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
@@ -354,6 +357,74 @@ describe('agent trigger delivery methods', () => {
).resolves.toMatchObject({ id: successor.delivery.id });
});
+ it('shields background completion work from workers that cannot resolve it', async () => {
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ }),
+ );
+ const claimInput = {
+ workerId: 'worker',
+ claimToken: 'claim',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ };
+
+ await expect(methods.claimNextAgentTriggerDelivery(claimInput)).resolves.toBeNull();
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ ...claimInput,
+ workerId: 'background-capable-worker',
+ claimToken: 'background-capable-claim',
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1],
+ }),
+ ).resolves.toMatchObject({ id: queued.delivery.id });
+ });
+
+ it('renews and classifies process-local completion producer liveness idempotently', async () => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const initialLease = new Date(START.getTime() + 30_000);
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-producer-lease',
+ envelope: { event: { source } },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ producerLeaseUntil: initialLease,
+ }),
+ );
+
+ await expect(
+ methods.getAgentTriggerDeliveryProducerLease({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ now: START,
+ }),
+ ).resolves.toEqual({ status: 'live', leaseUntil: initialLease });
+ await expect(
+ methods.getAgentTriggerDeliveryProducerLease({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ now: new Date(initialLease.getTime() + 1),
+ }),
+ ).resolves.toEqual({ status: 'expired', leaseUntil: initialLease });
+
+ const renewedUntil = new Date(START.getTime() + 60_000);
+ const renewal = {
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ leaseUntil: renewedUntil,
+ };
+ await expect(methods.renewAgentTriggerDeliveryProducerLease(renewal)).resolves.toBe(true);
+ await expect(methods.renewAgentTriggerDeliveryProducerLease(renewal)).resolves.toBe(true);
+ await expect(
+ methods.getAgentTriggerDeliveryProducerLease({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ now: initialLease,
+ }),
+ ).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil });
+ });
+
it('keeps capability-fenced work limited to capable workers through lease recovery', async () => {
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
@@ -491,6 +562,189 @@ describe('agent trigger delivery methods', () => {
).resolves.toMatchObject({ id: ordinary.delivery.id, status: 'leased' });
});
+ it('retires a failed internal completion admission and unblocks its lane successor', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'background-completion-lane';
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const first = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-1',
+ user,
+ orderingKey,
+ envelope: { event: { source } },
+ }),
+ );
+ const successor = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-2',
+ user,
+ orderingKey,
+ envelope: { event: { source } },
+ }),
+ );
+
+ const retirement = {
+ deliveryKey: first.delivery.deliveryKey,
+ sourceId: source.id,
+ settledAt: START,
+ reason: 'result persistence failed',
+ };
+ await expect(methods.retireAgentTriggerDelivery(retirement)).resolves.toBe(true);
+ await expect(methods.retireAgentTriggerDelivery(retirement)).resolves.toBe(true);
+ await expect(
+ methods.retireAgentTriggerDelivery({ ...retirement, sourceId: 'other-internal-source' }),
+ ).resolves.toBe(false);
+ await expect(
+ Delivery.findOne({ deliveryKey: first.delivery.deliveryKey }).lean(),
+ ).resolves.toMatchObject({
+ status: 'succeeded',
+ result: {
+ status: 'settled',
+ backgroundToolCompletionRetired: true,
+ reason: 'result persistence failed',
+ },
+ });
+
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'successor-worker',
+ claimToken: 'successor-claim',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ }),
+ ).resolves.toMatchObject({ id: successor.delivery.id });
+ });
+
+ it('does not retire a completion already leased by a resolver for a manual poll', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-leased',
+ user,
+ envelope: { event: { source } },
+ }),
+ );
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'completion-worker',
+ claimToken: 'completion-claim',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ }),
+ ).resolves.toMatchObject({ id: queued.delivery.id });
+
+ await expect(
+ methods.retireAgentTriggerDelivery({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ settledAt: START,
+ reason: 'manual poll elected',
+ onlyIfUnclaimed: true,
+ }),
+ ).resolves.toBe(false);
+ await expect(
+ Delivery.findOne({ deliveryKey: queued.delivery.deliveryKey }).lean(),
+ ).resolves.toMatchObject({ status: 'leased', leaseBy: 'completion-worker' });
+ });
+
+ it('retires a capability-shielded completion before a private resolver claims it', async () => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-shielded-pending',
+ envelope: { event: { source } },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ }),
+ );
+ await expect(Delivery.findById(queued.delivery.id).lean()).resolves.toMatchObject({
+ status: 'leased',
+ capabilityStatus: 'pending',
+ });
+
+ await expect(
+ methods.retireAgentTriggerDelivery({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ settledAt: START,
+ reason: 'manual poll elected',
+ onlyIfUnclaimed: true,
+ }),
+ ).resolves.toBe(true);
+ await expect(Delivery.findById(queued.delivery.id).lean()).resolves.toMatchObject({
+ status: 'succeeded',
+ result: { backgroundToolCompletionRetired: true },
+ });
+ });
+
+ it('does not retire a capability completion after a private resolver claims it', async () => {
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-private-lease',
+ envelope: { event: { source } },
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ }),
+ );
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'background-capable-worker',
+ claimToken: 'background-capable-claim',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1],
+ }),
+ ).resolves.toMatchObject({ id: queued.delivery.id });
+
+ await expect(
+ methods.retireAgentTriggerDelivery({
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ settledAt: START,
+ reason: 'manual poll elected',
+ onlyIfUnclaimed: true,
+ }),
+ ).resolves.toBe(false);
+ await expect(Delivery.findById(queued.delivery.id).lean()).resolves.toMatchObject({
+ status: 'leased',
+ capabilityStatus: 'leased',
+ });
+ });
+
+ it('reconciles only an irreversibly dead completion for manual polling', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const source = { id: 'background-tool-completion', type: 'internal' };
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'background-completion-dead',
+ user,
+ envelope: { event: { source } },
+ }),
+ );
+ await Delivery.updateOne(
+ { _id: queued.delivery.id },
+ { $set: { status: 'dead', settledAt: START } },
+ );
+
+ const retirement = {
+ deliveryKey: queued.delivery.deliveryKey,
+ sourceId: source.id,
+ settledAt: new Date(START.getTime() + 1),
+ reason: 'manual poll recovered dead completion',
+ onlyIfDead: true,
+ } as const;
+ await expect(methods.retireAgentTriggerDelivery(retirement)).resolves.toBe(true);
+ /** A later claim-release retry must recognize the recovery receipt even
+ * though the delivery no longer has a dead status. */
+ await expect(methods.retireAgentTriggerDelivery(retirement)).resolves.toBe(true);
+ await expect(
+ Delivery.findOne({ deliveryKey: queued.delivery.deliveryKey }).lean(),
+ ).resolves.toMatchObject({
+ status: 'succeeded',
+ result: { backgroundToolCompletionRetired: true },
+ });
+ });
+
it('claims older capability work before newer ordinary work', async () => {
const capability = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
index a9ae5add5f..49b9aa8699 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -16,7 +16,10 @@ import type {
IAgentTriggerUserPurge,
IAgentTriggerUserPurgeDocument,
} from '~/types/triggerDelivery';
-import { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from '~/types/triggerDelivery';
+import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+} from '~/types/triggerDelivery';
import { createIndexesWithRetry } from '~/utils/retry';
import logger from '~/config/winston';
@@ -104,8 +107,14 @@ export interface EnqueueAgentTriggerDeliveryInput {
coalesceUntil?: Date;
awaitTerminalHandling?: boolean;
requiredWorkerCapability?: string;
+ producerLeaseUntil?: Date;
}
+export type AgentTriggerProducerLeaseStatus =
+ | { status: 'live'; leaseUntil: Date }
+ | { status: 'expired'; leaseUntil: Date }
+ | { status: 'missing' };
+
export interface AgentTriggerDeliveryFence {
id: string;
workerId: string;
@@ -236,6 +245,24 @@ export interface AgentTriggerDeliveryMethods {
awaitTerminalHandling?: true;
},
) => Promise<boolean>;
+ retireAgentTriggerDelivery: (input: {
+ deliveryKey: string;
+ sourceId: string;
+ settledAt: Date;
+ reason: string;
+ onlyIfUnclaimed?: boolean;
+ onlyIfDead?: boolean;
+ }) => Promise<boolean>;
+ renewAgentTriggerDeliveryProducerLease: (input: {
+ deliveryKey: string;
+ sourceId: string;
+ leaseUntil: Date;
+ }) => Promise<boolean>;
+ getAgentTriggerDeliveryProducerLease: (input: {
+ deliveryKey: string;
+ sourceId: string;
+ now: Date;
+ }) => Promise<AgentTriggerProducerLeaseStatus>;
settleAgentTriggerHandlingOutcome: (
input: SettleAgentTriggerHandlingOutcomeInput,
) => Promise<boolean>;
@@ -892,7 +919,8 @@ export function createAgentTriggerDeliveryMethods(
): Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }> {
if (
input.requiredWorkerCapability != null &&
- input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1
+ input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
+ input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1
) {
throw new TypeError('Agent trigger delivery requires an unsupported worker capability');
}
@@ -1891,6 +1919,192 @@ export function createAgentTriggerDeliveryMethods(
return true;
}
+ /** Retires an internally pre-admitted delivery when its producer can prove
+ * that the result will never become dispatchable. This transition is keyed
+ * by the immutable delivery identity rather than a worker lease so the
+ * producer can unblock the lane even while a resolver is deferring it. */
+ async function retireAgentTriggerDelivery(input: {
+ deliveryKey: string;
+ sourceId: string;
+ settledAt: Date;
+ reason: string;
+ onlyIfUnclaimed?: boolean;
+ onlyIfDead?: boolean;
+ }): Promise<boolean> {
+ if (
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 256 ||
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256 ||
+ Number.isNaN(input.settledAt.getTime()) ||
+ (input.onlyIfUnclaimed === true && input.onlyIfDead === true)
+ ) {
+ throw new TypeError('Invalid agent trigger delivery retirement');
+ }
+ const result = {
+ status: 'settled',
+ backgroundToolCompletionRetired: true,
+ reason: input.reason.slice(0, MAX_ERROR_MESSAGE_LENGTH),
+ };
+ const statusFence = (() => {
+ if (input.onlyIfDead === true) {
+ return {
+ $or: [
+ { status: { $in: ['dead', 'capability_dead'] } },
+ { status: 'leased', capabilityStatus: 'dead' },
+ ],
+ };
+ }
+ if (input.onlyIfUnclaimed === true) {
+ return {
+ $or: [
+ { status: { $in: ['pending', 'capability_pending'] } },
+ /** A capability-shielded pending row looks leased to legacy
+ * workers but has no private claimant yet. */
+ { status: 'leased', capabilityStatus: 'pending' },
+ ],
+ };
+ }
+ return {
+ status: {
+ $in: ['pending', 'leased', 'capability_pending', 'capability_leased'],
+ },
+ };
+ })();
+ const retired = await Delivery()
+ .findOneAndUpdate(
+ {
+ deliveryKey: input.deliveryKey,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ ...statusFence,
+ },
+ {
+ $set: {
+ status: 'succeeded',
+ result,
+ settledAt: input.settledAt,
+ expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
+ laneCleanupPendingAt: input.settledAt,
+ },
+ $unset: {
+ leaseBy: 1,
+ leaseUntil: 1,
+ claimToken: 1,
+ capabilityStatus: 1,
+ claimAvailableAt: 1,
+ capabilityLeaseBy: 1,
+ capabilityLeaseUntil: 1,
+ capabilityClaimToken: 1,
+ lastError: 1,
+ },
+ },
+ { new: true },
+ )
+ .select('_id orderingKey laneCleanupPendingAt')
+ .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
+ if (retired?._id != null) {
+ try {
+ await fulfillLaneCleanupRequest(retired);
+ } catch (error) {
+ logger.warn('[agent-triggers] failed to finalize a retired internal delivery', {
+ deliveryKey: input.deliveryKey,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ return true;
+ }
+ return (
+ (await Delivery().exists({
+ deliveryKey: input.deliveryKey,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ status: 'succeeded',
+ 'result.backgroundToolCompletionRetired': true,
+ })) != null
+ );
+ }
+
+ /** Refreshes process-owner liveness without changing delivery claim state.
+ * The max transition makes a lost write receipt safe to replay and prevents
+ * an older heartbeat from shortening a newer lease. */
+ async function renewAgentTriggerDeliveryProducerLease(input: {
+ deliveryKey: string;
+ sourceId: string;
+ leaseUntil: Date;
+ }): Promise<boolean> {
+ if (
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 256 ||
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256 ||
+ !(input.leaseUntil instanceof Date) ||
+ !Number.isFinite(input.leaseUntil.getTime())
+ ) {
+ throw new TypeError('Invalid agent trigger producer lease renewal');
+ }
+ const renewed = await Delivery().updateOne(
+ {
+ deliveryKey: input.deliveryKey,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ status: { $in: ['pending', 'leased', 'capability_pending', 'capability_leased'] },
+ capabilityStatus: { $ne: 'dead' },
+ settledAt: { $exists: false },
+ },
+ [
+ {
+ $set: {
+ producerLeaseUntil: {
+ $cond: [
+ { $gt: ['$producerLeaseUntil', input.leaseUntil] },
+ '$producerLeaseUntil',
+ input.leaseUntil,
+ ],
+ },
+ },
+ },
+ ],
+ { timestamps: false },
+ );
+ return renewed.matchedCount === 1;
+ }
+
+ /** Reads only the private producer lease. Missing is intentionally distinct
+ * for compatibility with rows admitted before this evidence existed. */
+ async function getAgentTriggerDeliveryProducerLease(input: {
+ deliveryKey: string;
+ sourceId: string;
+ now: Date;
+ }): Promise<AgentTriggerProducerLeaseStatus> {
+ if (
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 256 ||
+ input.sourceId.length === 0 ||
+ input.sourceId.length > 256 ||
+ !(input.now instanceof Date) ||
+ !Number.isFinite(input.now.getTime())
+ ) {
+ throw new TypeError('Invalid agent trigger producer lease lookup');
+ }
+ const delivery = await Delivery()
+ .findOne({
+ deliveryKey: input.deliveryKey,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ })
+ .select('+producerLeaseUntil')
+ .lean<Pick<IAgentTriggerDelivery, 'producerLeaseUntil'>>();
+ if (delivery?.producerLeaseUntil == null) {
+ return { status: 'missing' };
+ }
+ return delivery.producerLeaseUntil.getTime() > input.now.getTime()
+ ? { status: 'live', leaseUntil: delivery.producerLeaseUntil }
+ : { status: 'expired', leaseUntil: delivery.producerLeaseUntil };
+ }
+
async function settleAgentTriggerHandlingOutcome(
input: SettleAgentTriggerHandlingOutcomeInput,
): Promise<boolean> {
@@ -3260,6 +3474,9 @@ export function createAgentTriggerDeliveryMethods(
beginAgentTriggerDeliveryAttempt,
deferAgentTriggerDeliveryAttempt,
completeAgentTriggerDelivery,
+ retireAgentTriggerDelivery,
+ renewAgentTriggerDeliveryProducerLease,
+ getAgentTriggerDeliveryProducerLease,
settleAgentTriggerHandlingOutcome,
admitAgentEventActorAction,
releaseAgentEventActorAction,
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
index 52e0894778..34cb661587 100644
--- a/packages/data-schemas/src/schema/triggerDelivery.ts
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -145,6 +145,8 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
capabilityLeaseBy: { type: String },
capabilityLeaseUntil: { type: Date },
capabilityClaimToken: { type: String },
+ /** Private process-owner heartbeat; never projected to legacy consumers. */
+ producerLeaseUntil: { type: Date, select: false },
attempts: { type: Number, required: true, default: 0, min: 0 },
availableAt: { type: Date, required: true },
envelopeBytes: { type: Number, min: 0 },
diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts
index e6cb71cd0f..b8c68a0ddf 100644
--- a/packages/data-schemas/src/types/triggerDelivery.ts
+++ b/packages/data-schemas/src/types/triggerDelivery.ts
@@ -13,6 +13,8 @@ export type AgentTriggerDeliveryStatus =
| 'capability_dead'
| 'dead';
export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1';
+export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
+ 'background_tool_completion_v1';
export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';
export interface AgentTriggerHandlingState {
@@ -103,6 +105,8 @@ export interface IAgentTriggerDelivery {
capabilityLeaseBy?: string;
capabilityLeaseUntil?: Date;
capabilityClaimToken?: string;
+ /** Durable liveness evidence for process-owned capability work. */
+ producerLeaseUntil?: Date;
attempts: number;
availableAt: Date;
envelopeBytes?: number;