Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🪇 perf: Coalesce Ready Background Completions (#16650)

lia-by-librechat[bot]f88f3fb2febb
diff --git a/api/server/experimental.js b/api/server/experimental.js
index 18983dafb8..2ad6e559fa 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -709,6 +709,8 @@ if (cluster.isMaster) {
address: server.address(),
completionResultBatchSize:
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
+ completionReceiptBatching:
+ baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling,
});
} catch (initErr) {
diff --git a/api/server/index.js b/api/server/index.js
index eec43ca22e..4ceba2caff 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -508,6 +508,8 @@ const startServer = async () => {
address: server.address(),
completionResultBatchSize:
appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
+ completionReceiptBatching:
+ appConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling,
});
const scheduleEngineArmed = (await initializeScheduleEngine()) != null;
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index f8944c6ea8..3977b8e3f8 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -6,6 +6,7 @@ const {
createSubagentCompletionWakeupResolver,
SUBAGENT_COMPLETION_SOURCE,
createBackgroundToolCompletionWakeupResolver,
+ createBackgroundToolDeadClaimRecovery,
BACKGROUND_TOOL_COMPLETION_SOURCE,
createAgentQueuedTurnLifecycle,
AGENT_QUEUED_TURN_SOURCE,
@@ -30,6 +31,22 @@ const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupReso
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),
+ getGenerationAdmissionEvidence,
+ recoverDeadClaim: createBackgroundToolDeadClaimRecovery(
+ (...args) => service.retire(...args),
+ methods.releaseBackgroundToolResultClaims,
+ (conversationId) => GenerationJobManager.getJob(conversationId),
+ ({ userId, conversationId, claimId }) =>
+ GenerationJobManager.fenceGenerationClaimForRecovery(
+ userId,
+ claimId,
+ conversationId,
+ conversationId,
+ ),
+ methods.releaseAgentBackgroundToolResultClaims,
+ methods,
+ getGenerationAdmissionEvidence,
+ ),
getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),
});
const eventActorAdapter = createAgentEventContinueResolver({
@@ -95,6 +112,7 @@ module.exports = {
persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
expediteCompletionWakeups: service.expediteCompletionWakeups,
getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
+ getBackgroundCompletionReceiptBatching: service.getBackgroundCompletionReceiptBatching,
releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
drainAgentTriggerDeliveriesForUser: service.drainUser,
prepareAgentTriggerUserPurge: service.prepareUserPurge,
diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.js
index df3a09b2d6..ed99ccab28 100644
--- a/api/server/services/Agents/triggers.spec.js
+++ b/api/server/services/Agents/triggers.spec.js
@@ -19,6 +19,7 @@ const mockQueuedTurnLifecycle = {
jest.mock('@librechat/api', () => ({
createCheckpointDeletionReclaimer: jest.fn((getJobs) => () => getJobs('owner', 'tenant')),
+ createBackgroundToolDeadClaimRecovery: jest.fn(() => jest.fn()),
createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),
createAgentContinuationResolver: jest.fn(() => jest.fn()),
createAgentEventContinueResolver: jest.fn(() => jest.fn()),
diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js
index 94d1f52eec..88ea7c4fa8 100644
--- a/api/server/services/Endpoints/agents/backgroundCompletion.js
+++ b/api/server/services/Endpoints/agents/backgroundCompletion.js
@@ -1,5 +1,6 @@
const {
createBackgroundToolCompletionWakeupHandler,
+ GenerationJobManager,
createBackgroundToolDeadClaimRecovery,
createPendingBackgroundCompletions,
createBackgroundToolResultHandler,
@@ -7,12 +8,15 @@ const {
} = require('@librechat/api');
const {
listPendingAgentBackgroundToolCompletions,
+ getAgentBackgroundToolResultBatch,
+ confirmAgentBackgroundToolResultBatch,
listUndeliveredAgentTriggerTaskIds,
} = require('~/models');
const {
enqueueAgentTrigger,
persistAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
+ getBackgroundCompletionReceiptBatching,
releaseAgentBackgroundToolResultClaims,
renewAgentTriggerProducerLease,
retireAgentTrigger,
@@ -26,6 +30,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
(deliveryKey, sourceId, result) =>
persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
(deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }),
+ getBackgroundCompletionReceiptBatching,
);
const pendingBackgroundToolCompletions = createPendingBackgroundCompletions({
@@ -52,6 +57,8 @@ function createDeadBackgroundToolClaimRecovery(
getGenerationJob,
fenceGenerationClaim,
releaseAgentBackgroundToolResultClaims,
+ { getAgentBackgroundToolResultBatch, confirmAgentBackgroundToolResultBatch },
+ (...args) => GenerationJobManager.getGenerationAdmissionEvidence(...args),
);
}
diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts
index 571dca62f6..ead7e37788 100644
--- a/packages/api/src/agents/background.spec.ts
+++ b/packages/api/src/agents/background.spec.ts
@@ -53,6 +53,42 @@ async function waitForSubagentTaskToSettle(
throw new Error('Timed out waiting for the detached subagent task.');
}
+describe('manual reconciliation restoration', () => {
+ it('upgrades only the confirmed local owner and preserves the marker on replay', () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const created = registry.create({
+ userId: 'marker-user',
+ conversationId: 'marker-convo',
+ toolCallId: 'marker-call',
+ toolName: 'tool',
+ });
+ if ('atCapacity' in created) throw new Error('Unexpected capacity');
+ registry.complete('marker-user', 'marker-convo', created.task.id, { content: 'done' });
+ const claim = { kind: 'manual' as const, claimId: 'poll' };
+ expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(
+ 'acquired',
+ );
+ expect(
+ registry.claimResult('marker-user', 'marker-convo', created.task.id, {
+ ...claim,
+ claimId: 'foreign',
+ receiptReconciled: true,
+ }),
+ ).toBe('claimed');
+ expect(created.task.resultClaim?.receiptReconciled).toBeUndefined();
+ expect(
+ registry.claimResult('marker-user', 'marker-convo', created.task.id, {
+ ...claim,
+ receiptReconciled: true,
+ }),
+ ).toBe('replay');
+ expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(
+ 'replay',
+ );
+ expect(created.task.resultClaim?.receiptReconciled).toBe(true);
+ });
+});
+
describe('isBackgroundEligibleToolName', () => {
it('excludes direct-path, host-special, and machinery tools', () => {
for (const name of [
diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts
index f98f0b3406..43cfb15893 100644
--- a/packages/api/src/agents/background.ts
+++ b/packages/api/src/agents/background.ts
@@ -668,6 +668,7 @@ export interface BackgroundTask {
claimId: string;
claimedAt: number;
generationId?: string;
+ receiptReconciled?: true;
};
/** The declared tool may return a process-local live artifact. A terminal
* same-generation poll may therefore deliver from the local claim after it
@@ -1823,7 +1824,12 @@ export class BackgroundTaskRegistryClass {
userId: string,
conversationId: string,
taskId: string,
- claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string },
+ claim: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ generationId?: string;
+ receiptReconciled?: true;
+ },
): 'acquired' | 'replay' | 'claimed' | 'not_ready' {
const task = this.get(userId, conversationId, taskId);
if (task == null || task.status === 'running') {
@@ -1834,9 +1840,14 @@ export class BackgroundTaskRegistryClass {
task.updatedAt = Date.now();
return 'acquired';
}
- return task.resultClaim.kind === claim.kind && task.resultClaim.claimId === claim.claimId
- ? 'replay'
- : 'claimed';
+ if (task.resultClaim.kind !== claim.kind || task.resultClaim.claimId !== claim.claimId) {
+ return 'claimed';
+ }
+ if (claim.receiptReconciled === true && task.resultClaim.receiptReconciled !== true) {
+ task.resultClaim.receiptReconciled = true;
+ task.updatedAt = Date.now();
+ }
+ return 'replay';
}
releaseResultClaim(
@@ -2554,6 +2565,7 @@ export async function runCheckBackgroundTask(params: {
conversationId,
messageId: task.messageId,
claimId: existingClaim.claimId,
+ ...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),
...(existingClaim.kind === 'manual'
? {
kind: 'manual' as const,
@@ -2674,6 +2686,7 @@ export async function runCheckBackgroundTask(params: {
taskId,
{
kind: 'manual',
+ receiptReconciled: true,
claimId: invocationId,
...(params.generationId == null ? {} : { generationId: params.generationId }),
},
@@ -2744,6 +2757,7 @@ export async function runCheckBackgroundTask(params: {
taskId,
{
kind: 'manual',
+ receiptReconciled: true,
claimId: invocationId,
...(params.generationId == null ? {} : { generationId: params.generationId }),
},
@@ -2854,6 +2868,7 @@ export async function runCheckBackgroundTask(params: {
conversationId,
messageId: claimedResult.messageId,
claimId: existingClaim.claimId,
+ ...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),
...(existingClaim.kind === 'manual'
? {
kind: 'manual' as const,
@@ -3242,6 +3257,7 @@ export function getBackgroundCodeDelivery(params: {
kind: task.resultClaim.kind,
claimId: task.resultClaim.claimId,
claimedAt: new Date(task.resultClaim.claimedAt),
+ ...(task.resultClaim.receiptReconciled === true && { receiptReconciled: true }),
...(task.resultClaim.generationId == null
? {}
: { generationId: task.resultClaim.generationId }),
diff --git a/packages/api/src/agents/backgroundClaims.ts b/packages/api/src/agents/backgroundClaims.ts
index 6f92bca763..320aab3994 100644
--- a/packages/api/src/agents/backgroundClaims.ts
+++ b/packages/api/src/agents/backgroundClaims.ts
@@ -4,7 +4,11 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup'
/** Reconciles manual polling with the independent automatic-delivery receipt,
* including polls reconstructed after the process-local registry was lost. */
export async function claimBackgroundToolResult(
- methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,
+ methods: Pick<
+ MessageMethods,
+ 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'
+ > &
+ Partial<Pick<MessageMethods, 'confirmBackgroundToolResultClaim'>>,
getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],
input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],
): ReturnType<MessageMethods['claimBackgroundToolResults']> {
@@ -30,7 +34,7 @@ export async function claimBackgroundToolResult(
if (messageId == null) {
throw new Error('The background result claim has no parent message identity');
}
- const release = async () => {
+ const release = async (allowReconciled = false) => {
const released = await methods.releaseBackgroundToolResultClaims({
userId: input.userId,
conversationId: input.conversationId,
@@ -38,8 +42,10 @@ export async function claimBackgroundToolResult(
taskIds: messageClaim.results.map((result) => result.taskId),
kind: input.kind,
claimId: input.claimId,
+ ...(input.batchId != null && { batchId: input.batchId }),
+ ...(input.kind === 'manual' && { onlyIfUnreconciled: true }),
});
- if (!released) {
+ if (!released && !allowReconciled) {
throw new Error('The background result claim could not be released');
}
};
@@ -51,6 +57,26 @@ export async function claimBackgroundToolResult(
throw error;
}
if (receiptClaim == null || receiptClaim.claimId === input.claimId) {
+ if (input.kind === 'manual' && methods.confirmBackgroundToolResultClaim != null) {
+ try {
+ if (
+ !(await methods.confirmBackgroundToolResultClaim({
+ userId: input.userId,
+ conversationId: input.conversationId,
+ messageId,
+ taskId: input.taskId,
+ claimId: input.claimId,
+ }))
+ ) {
+ throw new Error('The reconciled background result claim was superseded');
+ }
+ } catch (error) {
+ // This CAS removes only speculative ownership. A committed or unknown
+ // handoff remains owned for generation-fenced recovery.
+ await release(true);
+ throw error;
+ }
+ }
return messageClaim;
}
await release();
diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts
index 8886ea89c0..e04f82a262 100644
--- a/packages/api/src/agents/backgroundCompletion.ts
+++ b/packages/api/src/agents/backgroundCompletion.ts
@@ -65,6 +65,9 @@ export interface BackgroundToolDeadClaimRecoveryInput {
conversationId: string;
messageId: string;
claimId: string;
+ batchId?: string;
+ /** Automatic repair cannot reopen a committed manual handoff. */
+ onlyIfUnreconciled?: true;
/** Omitted for the legacy automatic-wakeup recovery path. */
kind?: 'manual' | 'wakeup';
/** Required to prove that a manual claim's owning generation is no longer active. */
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts
new file mode 100644
index 0000000000..5b06bdcb73
--- /dev/null
+++ b/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts
@@ -0,0 +1,1479 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import {
+ createMethods,
+ createModels,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+} from '@librechat/data-schemas';
+import type { IMessage } from '@librechat/data-schemas';
+import type { AgentTriggerFetch } from './triggers/host';
+import {
+ createBackgroundToolCompletionWakeupResolver,
+ createBackgroundToolDeadClaimRecovery,
+ BACKGROUND_TOOL_COMPLETION_SOURCE,
+} from './backgroundCompletionWakeup';
+import {
+ backgroundTaskRegistry,
+ runCheckBackgroundTask,
+ getBackgroundCodeDelivery,
+} from './background';
+import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './triggers/envelope';
+import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
+import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
+import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
+import { createAgentTriggerExecutionHost } from './triggers/host';
+import { prepareAgentTriggerDelivery } from './triggers/delivery';
+import { claimBackgroundToolResult } from './backgroundClaims';
+
+let mongo: MongoMemoryServer;
+let methods: ReturnType<typeof createMethods>;
+let userId: string;
+const conversationId = 'receipt-replay';
+const parentMessageId = 'receipt-parent';
+const agentId = 'agent_parent_1';
+const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3;
+
+beforeAll(async () => {
+ mongo = await MongoMemoryServer.create();
+ await mongoose.connect(mongo.getUri());
+ createModels(mongoose);
+ methods = createMethods(mongoose);
+}, 60_000);
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongo.stop();
+});
+
+beforeEach(async () => {
+ await Promise.all(
+ ['AgentTriggerDelivery', 'AgentTriggerLaneSequence', 'Conversation', 'Message'].map((name) =>
+ mongoose.models[name].deleteMany({}),
+ ),
+ );
+ userId = new mongoose.Types.ObjectId().toString();
+ await mongoose.models.Conversation.create({
+ conversationId,
+ user: userId,
+ tenantId: 'tenant-1',
+ endpoint: 'agents',
+ agent_id: agentId,
+ });
+ await mongoose.models.Message.create({
+ messageId: parentMessageId,
+ conversationId,
+ user: userId,
+ tenantId: 'tenant-1',
+ parentMessageId: 'user-parent',
+ isCreatedByUser: false,
+ unfinished: false,
+ endpoint: 'agents',
+ content: [],
+ });
+});
+
+async function ready(taskId: string, persistReceipt = true) {
+ const envelope = createAgentTriggerEnvelope({
+ mode: 'continue',
+ requestId: taskId,
+ deliveryId: taskId,
+ receivedAt: Date.now(),
+ principal: { id: userId, tenantId: 'tenant-1' },
+ event: {
+ id: taskId,
+ type: 'background-tool.completion',
+ occurredAt: Date.now(),
+ source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' },
+ payload: { taskId, toolCallId: taskId, toolName: 'tool' },
+ },
+ target: { conversationId, parentMessageId, agentId },
+ input: 'waiting',
+ });
+ const row = prepareAgentTriggerDelivery(envelope, {
+ orderingKey: taskId,
+ requiredWorkerCapability: capability,
+ });
+ await methods.enqueueAgentTriggerDelivery(row);
+ if (persistReceipt)
+ await methods.persistAgentBackgroundToolResult({
+ deliveryKey: row.deliveryKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ result: { status: 'completed', output: taskId, settledAt: new Date() },
+ });
+ return envelope;
+}
+
+function owner(deliveryKey: string) {
+ return {
+ deliveryKey,
+ userId,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ tenantId: 'tenant-1',
+ conversationId,
+ parentMessageId,
+ agentId,
+ };
+}
+
+it('replays exactly the admitted input after a lost response and a definitely rejected retry', async () => {
+ const root = await ready('one');
+ const sibling = await ready('two');
+ const inputs: string[] = [];
+ const consumed: string[] = [];
+ let firstInput: string | undefined;
+ let calls = 0;
+ const fetcher: AgentTriggerFetch = async (_url, init) => {
+ const body = JSON.parse(String(init?.body)) as { text: string };
+ inputs.push(body.text);
+ if (++calls === 1) {
+ firstInput = body.text;
+ consumed.push('one', 'two');
+ throw Object.assign(new Error('lost admitted response'), { code: 'ECONNRESET' });
+ }
+ if (calls === 2)
+ throw Object.assign(new Error('retry did not connect'), { code: 'ECONNREFUSED' });
+ if (body.text !== firstInput) consumed.push('three');
+ return new Response(
+ JSON.stringify({
+ status: 'started',
+ streamId: conversationId,
+ conversationId,
+ generationCreatedAt: 100,
+ }),
+ { status: 200 },
+ );
+ };
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ const host = createAgentTriggerExecutionHost({
+ prepareContinue: resolve,
+ fetch: fetcher,
+ mintToken: () => 'test',
+ getBaseUrl: () => 'http://localhost',
+ });
+ const options = { requiredWorkerCapability: capability };
+ await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'ambiguous' });
+ const late = await ready('three');
+ await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'definite' });
+ await expect(host.dispatch(root, options)).resolves.toMatchObject({ status: 'started' });
+ expect(inputs).toEqual([firstInput, firstInput, firstInput]);
+ expect(consumed).toEqual(['one', 'two']);
+ await expect(host.dispatch(sibling, options)).resolves.toMatchObject({ status: 'settled' });
+ const lateEnvelope = late.mode === 'continue' ? late : undefined;
+ if (lateEnvelope == null) throw new Error('Expected continuation');
+ const prepared = await resolve(lateEnvelope, {
+ idempotencyKey: getAgentTriggerIdempotencyKey(late),
+ ...options,
+ });
+ expect(prepared?.status === 'ready' && prepared.input).toContain('three');
+});
+
+it('cleans a late predecessor projection without clearing its successor receipt batch', async () => {
+ const root = await ready('one');
+ const deliveryKey = getAgentTriggerIdempotencyKey(root);
+ const scope = owner(deliveryKey);
+ const old = await methods.claimAgentBackgroundToolResultBatch({
+ ...scope,
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ if (old.status !== 'acquired') throw new Error('Expected batch');
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...scope,
+ claimId: deliveryKey,
+ batchId: old.batchId,
+ }),
+ ).toBe(true);
+ await mongoose.models.Message.updateOne(
+ { messageId: parentMessageId, user: userId },
+ {
+ $set: {
+ content: [
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'one',
+ output: 'one',
+ backgroundTask: {
+ taskId: 'one',
+ toolName: 'tool',
+ status: 'completed',
+ completionWakeup: true,
+ completionReceipt: true,
+ },
+ },
+ },
+ ],
+ },
+ },
+ );
+ // A predecessor's delayed projection CAS lands after its cleanup completed.
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'wakeup',
+ claimId: deliveryKey,
+ batchId: old.batchId,
+ limit: 1,
+ });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ await expect(
+ resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }),
+ ).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING' });
+ const successor = await methods.getAgentBackgroundToolResultBatch(scope);
+ expect(successor?.batchId).not.toBe(old.batchId);
+ await expect(
+ resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }),
+ ).resolves.toMatchObject({ status: 'ready' });
+ expect((await methods.getAgentBackgroundToolResultBatch(scope))?.batchId).toBe(
+ successor?.batchId,
+ );
+});
+
+it('confirms a dead owner with native admission proof instead of re-presenting its results', async () => {
+ const root = await ready('one');
+ const sibling = await ready('two');
+ const deliveryKey = getAgentTriggerIdempotencyKey(root);
+ const scope = owner(deliveryKey);
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...scope,
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ await mongoose.models.AgentTriggerDelivery.updateOne(
+ { deliveryKey },
+ { $set: { capabilityStatus: 'dead' } },
+ );
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async (key, sourceId, reason, options) =>
+ methods.retireAgentTriggerDelivery({
+ deliveryKey: key,
+ sourceId,
+ reason,
+ settledAt: new Date(),
+ ...options,
+ }),
+ methods.releaseBackgroundToolResultClaims,
+ async () => null,
+ async () => 'started',
+ methods.releaseAgentBackgroundToolResultClaims,
+ methods,
+ async () => ({ generationId: conversationId, generationCreatedAt: 100 }),
+ );
+ expect(
+ await recover({ userId, conversationId, messageId: parentMessageId, claimId: deliveryKey }),
+ ).toBe(false);
+ const follower = await methods.claimAgentBackgroundToolResultBatch({
+ ...owner(getAgentTriggerIdempotencyKey(sibling)),
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ expect(follower).toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
+});
+
+async function project(taskId: string) {
+ await mongoose.models.Message.updateOne(
+ { messageId: parentMessageId, user: userId },
+ {
+ $push: {
+ content: {
+ type: 'tool_call',
+ tool_call: {
+ id: taskId,
+ output: `durable-${taskId}`,
+ backgroundTask: {
+ taskId,
+ toolName: 'tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ completionReceipt: true,
+ },
+ },
+ },
+ },
+ },
+ );
+}
+
+it('reconstructs a failed receipt write from the successful terminal message projection', async () => {
+ const root = await ready('one', false);
+ const deliveryKey = getAgentTriggerIdempotencyKey(root);
+ await project('one');
+ await mongoose.models.AgentTriggerDelivery.updateOne(
+ { deliveryKey },
+ { $set: { producerLeaseUntil: new Date(0) } },
+ );
+ const fetcher: AgentTriggerFetch = async (_url, init) => {
+ expect(JSON.parse(String(init?.body)).text).toContain('durable-one');
+ return new Response(
+ JSON.stringify({
+ status: 'started',
+ conversationId,
+ streamId: conversationId,
+ generationCreatedAt: 100,
+ }),
+ { status: 200 },
+ );
+ };
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ const host = createAgentTriggerExecutionHost({
+ prepareContinue: resolve,
+ fetch: fetcher,
+ mintToken: () => 'test',
+ getBaseUrl: () => 'http://localhost',
+ });
+ await expect(
+ host.dispatch(root, { requiredWorkerCapability: capability }),
+ ).resolves.toMatchObject({ status: 'started' });
+ expect(
+ await methods.getAgentBackgroundToolResult({
+ deliveryKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ }),
+ ).toMatchObject({ output: 'durable-one', resultClaim: { appliedAt: expect.any(Date) } });
+});
+
+it('does not settle a root when a speculative manual poll and automatic owner mutually yield', async () => {
+ const root = await ready('one');
+ await ready('two');
+ await project('one');
+ const key = getAgentTriggerIdempotencyKey(root);
+ let entered: () => void = () => undefined;
+ let captured: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const automaticEntered = new Promise<void>((resolve) => {
+ entered = resolve;
+ });
+ const snapshotTaken = new Promise<void>((resolve) => {
+ captured = resolve;
+ });
+ const manualMayYield = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ let lookups = 0;
+ const getter = async (input: Parameters<typeof methods.getAgentBackgroundToolResultClaim>[0]) => {
+ if (++lookups === 1) return methods.getAgentBackgroundToolResultClaim(input);
+ await automaticEntered;
+ const snapshot = await methods.getAgentBackgroundToolResultClaim(input);
+ captured();
+ await manualMayYield;
+ return snapshot;
+ };
+ const claim = methods.claimBackgroundToolResults.bind(methods);
+ const automatic = jest
+ .spyOn(methods, 'claimBackgroundToolResults')
+ .mockImplementation(async (input) => {
+ if (input.kind === 'wakeup' && input.taskId === 'one') {
+ entered();
+ await snapshotTaken;
+ }
+ return claim(input);
+ });
+ const manual = claimBackgroundToolResult(methods, getter, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ generationId: 'manual-generation',
+ });
+ // Allow the initial manual projection CAS to commit before receipt selection.
+ const deadline = Date.now() + 3000;
+ while (
+ (await mongoose.models.Message.findOne({
+ messageId: parentMessageId,
+ 'content.tool_call.backgroundTask.resultClaim.claimId': 'manual-poll',
+ }).lean()) == null
+ ) {
+ if (Date.now() > deadline) throw new Error('Manual claim did not arrive');
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ await expect(
+ resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability }),
+ ).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', deferWithoutAttempt: true });
+ resume();
+ expect(await manual).toMatchObject({ status: 'claimed' });
+ automatic.mockRestore();
+ const retry = await resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability });
+ expect(retry?.status === 'ready' && retry.input).toContain('one');
+ expect(retry?.status === 'ready' && retry.input).toContain('two');
+});
+
+it('settles a root only after a manual poll has completed receipt reconciliation', async () => {
+ const root = await ready('one');
+ await project('one');
+ expect(
+ (
+ await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ })
+ ).status,
+ ).toBe('acquired');
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ await expect(
+ resolve(root, {
+ idempotencyKey: getAgentTriggerIdempotencyKey(root),
+ requiredWorkerCapability: capability,
+ }),
+ ).resolves.toEqual({ status: 'settled' });
+});
+
+it.each([false, true])(
+ 'rolls back an unconfirmed manual claim with generation identity %s',
+ async (withGeneration) => {
+ const root = await ready('one');
+ await project('one');
+ const confirmation = jest
+ .spyOn(mongoose.models.Message, 'updateOne')
+ .mockImplementationOnce(() => {
+ throw new Error('confirmation unavailable');
+ });
+ await expect(
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ ...(withGeneration && { generationId: 'manual-generation' }),
+ }),
+ ).rejects.toThrow('confirmation unavailable');
+ confirmation.mockRestore();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ const prepared = await resolve(root, {
+ idempotencyKey: getAgentTriggerIdempotencyKey(root),
+ requiredWorkerCapability: capability,
+ });
+ expect(prepared?.status === 'ready' && prepared.input).toContain('"result":"one"');
+ },
+);
+
+it('resolves a committed manual confirmation whose write reply was lost', async () => {
+ const root = await ready('one');
+ await project('one');
+ const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message);
+ const lostReply = jest
+ .spyOn(mongoose.models.Message, 'updateOne')
+ .mockImplementationOnce((...args) => {
+ const query = update(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ await execute();
+ throw new Error('lost confirmation reply');
+ });
+ return query;
+ });
+ const consumed = await claimBackgroundToolResult(
+ methods,
+ methods.getAgentBackgroundToolResultClaim,
+ {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ generationId: 'manual-generation',
+ },
+ );
+ expect(consumed).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });
+ lostReply.mockRestore();
+ expect(
+ await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskIds: ['one'],
+ kind: 'manual',
+ claimId: 'manual-poll',
+ onlyIfUnreconciled: true,
+ }),
+ ).toBe(false);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ await expect(
+ resolve(root, {
+ idempotencyKey: getAgentTriggerIdempotencyKey(root),
+ requiredWorkerCapability: capability,
+ }),
+ ).resolves.toEqual({ status: 'settled' });
+});
+
+it('recovers abandoned manual ownership after both confirmation and rollback failed', async () => {
+ const root = await ready('one');
+ await ready('two');
+ await project('one');
+ const confirmation = jest
+ .spyOn(methods, 'confirmBackgroundToolResultClaim')
+ .mockRejectedValueOnce(new Error('confirmation unavailable'));
+ const rollback = jest
+ .spyOn(methods, 'releaseBackgroundToolResultClaims')
+ .mockRejectedValueOnce(new Error('rollback unavailable'));
+ await expect(
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ generationId: 'manual-generation',
+ }),
+ ).rejects.toThrow('rollback unavailable');
+ confirmation.mockRestore();
+ rollback.mockRestore();
+ let active = true;
+ const getGenerationJob = async () =>
+ active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null;
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async () => false,
+ methods.releaseBackgroundToolResultClaims,
+ getGenerationJob,
+ async () => 'unavailable',
+ );
+ const recoverDeadClaim = jest.fn(recover);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob,
+ recoverDeadClaim,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ const key = getAgentTriggerIdempotencyKey(root);
+ await mongoose.models.AgentTriggerDelivery.updateOne(
+ { deliveryKey: key },
+ { $set: { capabilityClaimToken: 'exact-queue-lease' } },
+ );
+ const context = {
+ idempotencyKey: key,
+ requiredWorkerCapability: capability,
+ deliveryClaimToken: 'exact-queue-lease',
+ };
+ await expect(resolve(root, context)).rejects.toMatchObject({ code: 'PARENT_NOT_READY' });
+ expect(recoverDeadClaim).not.toHaveBeenCalled();
+ active = false;
+ await expect(resolve(root, context)).rejects.toMatchObject({
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ deferWithoutAttempt: true,
+ });
+ expect(recoverDeadClaim).toHaveBeenCalledWith(
+ expect.objectContaining({
+ kind: 'manual',
+ claimId: 'manual-poll',
+ generationId: 'manual-generation',
+ }),
+ );
+ const prepared = await resolve(root, context);
+ expect(prepared?.status === 'ready' && prepared.input).toContain('one');
+ expect(prepared?.status === 'ready' && prepared.input).toContain('two');
+});
+
+it('does not use a different manual owner as confirmation read-back proof', async () => {
+ await ready('one');
+ await project('one');
+ const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message);
+ const replaced = jest
+ .spyOn(mongoose.models.Message, 'updateOne')
+ .mockImplementationOnce((...args) => {
+ const query = update(...args);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ await update(
+ { user: userId, messageId: parentMessageId },
+ {
+ $set: {
+ 'content.0.tool_call.backgroundTask.resultClaim': {
+ kind: 'manual',
+ claimId: 'successor-poll',
+ receiptReconciled: true,
+ },
+ },
+ },
+ );
+ throw new Error('predecessor confirmation failed');
+ });
+ return query;
+ });
+ await expect(
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ }),
+ ).rejects.toThrow('predecessor confirmation failed');
+ replaced.mockRestore();
+ expect(
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'wakeup',
+ claimId: 'automatic',
+ }),
+ ).toMatchObject({
+ status: 'claimed',
+ claim: { claimId: 'successor-poll', receiptReconciled: true },
+ });
+});
+
+it.each(['missing-receipt', 'batch-root', 'batch-sibling'] as const)(
+ 'preserves a manual handoff committed after the %s recovery snapshot',
+ async (scenario) => {
+ const polled = await ready('one', scenario !== 'missing-receipt');
+ const other = scenario === 'batch-sibling' ? await ready('two') : undefined;
+ const root = other ?? polled;
+ await project('one');
+ let confirmationReached: () => void = () => undefined;
+ let confirmNow: () => void = () => undefined;
+ let recoveryReached: () => void = () => undefined;
+ let recoverNow: () => void = () => undefined;
+ const confirmationEntered = new Promise<void>((resolve) => {
+ confirmationReached = resolve;
+ });
+ const confirmationBarrier = new Promise<void>((resolve) => {
+ confirmNow = resolve;
+ });
+ const recoveryEntered = new Promise<void>((resolve) => {
+ recoveryReached = resolve;
+ });
+ const recoveryBarrier = new Promise<void>((resolve) => {
+ recoverNow = resolve;
+ });
+ const confirm = methods.confirmBackgroundToolResultClaim.bind(methods);
+ const pausedConfirmation = jest
+ .spyOn(methods, 'confirmBackgroundToolResultClaim')
+ .mockImplementationOnce(async (input) => {
+ confirmationReached();
+ await confirmationBarrier;
+ return confirm(input);
+ });
+ const manual = claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ generationId: 'manual-generation',
+ });
+ await confirmationEntered;
+ let manualActive = true;
+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async () => false,
+ release,
+ async () =>
+ manualActive
+ ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } }
+ : null,
+ async () => 'unavailable',
+ );
+ const recoverDeadClaim = jest.fn(async (input: Parameters<typeof recover>[0]) => {
+ recoveryReached();
+ await recoveryBarrier;
+ return recover(input);
+ });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ recoverDeadClaim,
+ });
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ const key = getAgentTriggerIdempotencyKey(root);
+ await mongoose.models.AgentTriggerDelivery.updateMany(
+ { deliveryKey: { $in: [key, getAgentTriggerIdempotencyKey(polled)] } },
+ {
+ $set: {
+ capabilityStatus: 'leased',
+ capabilityClaimToken: 'exact-queue-lease',
+ capabilityLeaseBy: 'automatic-worker',
+ capabilityLeaseUntil: new Date(Date.now() + 60_000),
+ },
+ },
+ );
+ const context = {
+ idempotencyKey: key,
+ requiredWorkerCapability: capability,
+ deliveryClaimToken: 'exact-queue-lease',
+ };
+ const automatic = resolve(root, context);
+ await recoveryEntered;
+ confirmNow();
+ expect(await manual).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });
+ // The automatic queue lease won manual retirement. The poll returned its
+ // result and its generation ended while recovery held a stale snapshot.
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: getAgentTriggerIdempotencyKey(polled),
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ settledAt: new Date(),
+ reason: 'manual poll',
+ onlyIfUnclaimed: true,
+ requireTransition: true,
+ }),
+ ).toBe(false);
+ manualActive = false;
+ recoverNow();
+ await expect(automatic).rejects.toMatchObject({
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ deferWithoutAttempt: true,
+ });
+ expect(recoverDeadClaim).toHaveBeenCalledWith(
+ expect.objectContaining({ onlyIfUnreconciled: true }),
+ );
+ expect(release).toHaveBeenCalledWith(
+ expect.objectContaining({ kind: 'manual', claimId: 'manual-poll', onlyIfUnreconciled: true }),
+ );
+ const claim = await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'wakeup',
+ claimId: 'probe',
+ });
+ expect(claim).toMatchObject({
+ status: 'claimed',
+ claim: { claimId: 'manual-poll', receiptReconciled: true },
+ });
+ const pollEnvelope = polled.mode === 'continue' ? polled : undefined;
+ if (pollEnvelope == null) throw new Error('Expected polled continuation');
+ await expect(
+ resolve(pollEnvelope, {
+ idempotencyKey: getAgentTriggerIdempotencyKey(polled),
+ requiredWorkerCapability: capability,
+ }),
+ ).resolves.toEqual({ status: 'settled' });
+ if (other != null) {
+ const row = await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(polled));
+ if (row == null) throw new Error('Expected polled delivery');
+ expect(
+ await methods.completeAgentTriggerDelivery({
+ id: row.id,
+ workerId: 'automatic-worker',
+ claimToken: 'exact-queue-lease',
+ attempt: 1,
+ settledAt: new Date(),
+ result: { status: 'settled', mode: 'continue', conversationId },
+ }),
+ ).toBe(true);
+ const remaining = await resolve(root, context);
+ if (remaining?.status !== 'ready') throw new Error('Expected undelivered sibling');
+ expect(remaining.input).toContain('"background_task_id":"two"');
+ expect(remaining.input).not.toContain('"background_task_id":"one"');
+ await remaining.beginDispatch?.();
+ }
+ pausedConfirmation.mockRestore();
+ },
+);
+
+it('keeps explicit manual recovery able to reopen a committed claim after its generation ends', async () => {
+ await ready('one');
+ await project('one');
+ await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ generationId: 'manual-generation',
+ });
+ let active = true;
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async () => false,
+ methods.releaseBackgroundToolResultClaims,
+ async () =>
+ active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null,
+ async () => 'unavailable',
+ );
+ const input = {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ claimId: 'manual-poll',
+ kind: 'manual' as const,
+ generationId: 'manual-generation',
+ };
+ expect(await recover(input)).toBe(false);
+ active = false;
+ expect(await recover({ ...input, onlyIfUnreconciled: true })).toBe(false);
+ expect(await recover(input)).toBe(true);
+ expect(
+ (
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'manual',
+ claimId: 'next-poll',
+ })
+ ).status,
+ ).toBe('acquired');
+});
+
+it.each(['same-generation', 'later-generation'] as const)(
+ 'restores manual reconciliation after %s final persistence',
+ async (scenario) => {
+ const generationId = scenario === 'same-generation' ? parentMessageId : 'manual-generation';
+ const created = backgroundTaskRegistry.create({
+ userId,
+ conversationId,
+ toolCallId: 'manual-final-call',
+ toolName: 'tool',
+ messageId: parentMessageId,
+ harvestStarted: true,
+ });
+ if ('atCapacity' in created) throw new Error('Unexpected task capacity');
+ const taskId = created.task.id;
+ const root = await ready(taskId);
+ await project(taskId);
+ await mongoose.models.Message.updateOne(
+ { user: userId, messageId: parentMessageId },
+ { $set: { unfinished: true } },
+ );
+ const original = await mongoose.models.Message.findOne({
+ user: userId,
+ messageId: parentMessageId,
+ }).lean<Pick<IMessage, 'content'>>();
+ backgroundTaskRegistry.complete(userId, conversationId, taskId, { content: 'manual-result' });
+ backgroundTaskRegistry.finishHarvest(userId, conversationId, taskId);
+ backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, taskId, {
+ renew: async () => true,
+ retire: async (reason, options) =>
+ methods.retireAgentTriggerDelivery({
+ deliveryKey: getAgentTriggerIdempotencyKey(root),
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ settledAt: new Date(),
+ reason,
+ ...options,
+ }),
+ });
+ const polled = JSON.parse(
+ await runCheckBackgroundTask({
+ userId,
+ conversationId,
+ args: { background_task_id: taskId },
+ toolCallId: 'confirmed-poll',
+ runId: 'manual-run',
+ generationId,
+ claimBackgroundToolResult: (input) =>
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),
+ }),
+ );
+ expect(polled).toMatchObject({ status: 'completed', result: 'manual-result' });
+ expect(backgroundTaskRegistry.get(userId, conversationId, taskId)?.resultClaim).toMatchObject({
+ receiptReconciled: true,
+ });
+ // Final full persistence uses content captured before manual confirmation.
+ await mongoose.models.Message.updateOne(
+ { user: userId, messageId: parentMessageId },
+ { $set: { unfinished: false, content: original?.content } },
+ );
+ const restored = getBackgroundCodeDelivery({
+ userId,
+ conversationId,
+ args: { background_task_id: taskId },
+ });
+ expect(restored?.backgroundTask?.resultClaim).toMatchObject({ receiptReconciled: true });
+ await methods.updateToolCallResult({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ toolCallId: taskId,
+ output: 'manual-result',
+ backgroundTask: restored?.backgroundTask,
+ });
+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async () => false,
+ release,
+ async () => null,
+ async () => 'unavailable',
+ );
+ const claim = await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId,
+ kind: 'wakeup',
+ claimId: 'automatic-probe',
+ });
+ if (claim.status !== 'claimed' || claim.claim == null)
+ throw new Error('Expected preserved manual claim');
+ expect(claim.claim).toMatchObject({ receiptReconciled: true });
+ expect(
+ await recover({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ claimId: claim.claim.claimId,
+ kind: 'manual',
+ generationId,
+ onlyIfUnreconciled: true,
+ }),
+ ).toBe(false);
+ expect(
+ (await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(root)))?.status,
+ ).toBe('succeeded');
+ },
+);
+
+it.each(['admitted', 'unpublished'] as const)(
+ 'uses native recovery proof after custom-store %s job cleanup',
+ async (state) => {
+ const root = await ready('one');
+ const sibling = await ready('two');
+ const key = getAgentTriggerIdempotencyKey(root);
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...owner(key),
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ // Historical lookup is optional; the native claim/fence CAS still exists.
+ Reflect.set(store, 'getIdempotencyClaim', undefined);
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ try {
+ const claimed = await manager.claimGeneration(userId, key, conversationId, conversationId, 2);
+ if (state === 'admitted') {
+ const job = await manager.createJob(conversationId, userId, conversationId, {
+ idempotencyClientRequestId: key,
+ idempotencyClaimToken: claimed.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(await store.deleteJob(conversationId, job.createdAt)).toBe(true);
+ }
+ expect(
+ await manager.getGenerationAdmissionEvidence(userId, key, conversationId, conversationId),
+ ).toBeNull();
+ await mongoose.models.AgentTriggerDelivery.updateOne(
+ { deliveryKey: key },
+ { $set: { capabilityStatus: 'dead' } },
+ );
+ const release = jest.fn(methods.releaseAgentBackgroundToolResultClaims);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async (deliveryKey, sourceId, reason, options) =>
+ methods.retireAgentTriggerDelivery({
+ deliveryKey,
+ sourceId,
+ reason,
+ settledAt: new Date(),
+ ...options,
+ }),
+ methods.releaseBackgroundToolResultClaims,
+ async () => null,
+ ({ userId, conversationId, claimId }) =>
+ manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId),
+ release,
+ methods,
+ (...args) => manager.getGenerationAdmissionEvidence(...args),
+ );
+ expect(
+ await recover({ userId, conversationId, messageId: parentMessageId, claimId: key }),
+ ).toBe(state === 'unpublished');
+ if (state === 'admitted') {
+ expect(release).not.toHaveBeenCalled();
+ expect(
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...owner(getAgentTriggerIdempotencyKey(sibling)),
+ limit: 8,
+ maxMetadataChars: 16000,
+ }),
+ ).toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
+ const batch = await methods.getAgentBackgroundToolResultBatch(owner(key));
+ expect(batch?.proofCopiedAt).toBeInstanceOf(Date);
+ } else {
+ expect(release).toHaveBeenCalledWith(expect.objectContaining({ recoveryFenced: true }));
+ expect(
+ (
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...owner(getAgentTriggerIdempotencyKey(sibling)),
+ limit: 8,
+ maxMetadataChars: 16000,
+ })
+ ).status,
+ ).toBe('acquired');
+ }
+ } finally {
+ await manager.destroy();
+ }
+ },
+);
+
+it.each(['lookup', 'confirmation', 'confirmation-unknown'] as const)(
+ 'preserves a committed manual replay when %s is unavailable',
+ async (phase) => {
+ const root = await ready('one');
+ await project('one');
+ const key = getAgentTriggerIdempotencyKey(root);
+ await mongoose.models.AgentTriggerDelivery.updateOne(
+ { deliveryKey: key },
+ { $set: { capabilityStatus: 'leased', capabilityClaimToken: 'automatic-lease' } },
+ );
+ const input = {
+ userId,
+ conversationId,
+ taskId: 'one',
+ kind: 'manual' as const,
+ claimId: 'committed-poll',
+ generationId: 'manual-generation',
+ };
+ const delivered = await claimBackgroundToolResult(
+ methods,
+ methods.getAgentBackgroundToolResultClaim,
+ input,
+ );
+ expect(delivered).toMatchObject({ status: 'acquired' });
+ let restore: () => void = () => undefined;
+ try {
+ if (phase === 'lookup') {
+ await expect(
+ claimBackgroundToolResult(
+ methods,
+ async () => {
+ throw new Error('receipt lookup unavailable');
+ },
+ input,
+ ),
+ ).rejects.toThrow();
+ } else if (phase === 'confirmation') {
+ const failure = jest
+ .spyOn(mongoose.models.Message, 'updateOne')
+ .mockImplementationOnce(() => {
+ throw new Error('confirmation write unavailable');
+ });
+ restore = () => {
+ failure.mockRestore();
+ };
+ // Exact committed read-back succeeds even though this replay's write fails.
+ await expect(
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ ...input,
+ generationId: undefined,
+ }),
+ ).resolves.toEqual(delivered);
+ } else {
+ const failure = jest
+ .spyOn(methods, 'confirmBackgroundToolResultClaim')
+ .mockRejectedValueOnce(new Error('confirmation outcome unavailable'));
+ restore = () => {
+ failure.mockRestore();
+ };
+ await expect(
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),
+ ).rejects.toThrow('confirmation outcome unavailable');
+ }
+ } finally {
+ restore();
+ }
+ const competitor = await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'one',
+ kind: 'wakeup',
+ claimId: 'automatic-probe',
+ });
+ expect(competitor).toMatchObject({
+ status: 'claimed',
+ claim: { claimId: input.claimId, generationId: 'manual-generation', receiptReconciled: true },
+ });
+ expect(
+ await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskIds: ['one'],
+ kind: 'manual',
+ claimId: input.claimId,
+ onlyIfUnreconciled: true,
+ }),
+ ).toBe(false);
+ if (root.mode !== 'continue') throw new Error('Expected continuation');
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ await expect(
+ resolve(root, {
+ idempotencyKey: key,
+ requiredWorkerCapability: capability,
+ deliveryClaimToken: 'automatic-lease',
+ }),
+ ).resolves.toEqual({ status: 'settled' });
+ },
+);
+
+it('repairs native started-fence confirmation from an applied follower after a retired owner crashed', async () => {
+ const root = await ready('one');
+ const sibling = await ready('two');
+ const key = getAgentTriggerIdempotencyKey(root);
+ const claimed = await methods.claimAgentBackgroundToolResultBatch({
+ ...owner(key),
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ if (claimed.status !== 'acquired') throw new Error('Expected frozen batch');
+ await mongoose.models.AgentTriggerDelivery.updateOne(
+ { deliveryKey: key },
+ { $set: { capabilityStatus: 'dead' } },
+ );
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async (deliveryKey, sourceId, reason, options) =>
+ methods.retireAgentTriggerDelivery({
+ deliveryKey,
+ sourceId,
+ reason,
+ settledAt: new Date(),
+ ...options,
+ }),
+ methods.releaseBackgroundToolResultClaims,
+ async () => null,
+ async () => 'started',
+ methods.releaseAgentBackgroundToolResultClaims,
+ methods,
+ async () => null,
+ );
+ const Delivery = mongoose.models.AgentTriggerDelivery;
+ const copyFailure = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('proof copy lost');
+ });
+ await expect(
+ recover({ userId, conversationId, messageId: parentMessageId, claimId: key }),
+ ).rejects.toThrow('proof copy lost');
+ copyFailure.mockRestore();
+ expect(await Delivery.findOne({ deliveryKey: key }).lean()).toMatchObject({
+ status: 'succeeded',
+ });
+ expect(await Delivery.findOne({ deliveryKey: key }).lean()).not.toHaveProperty('expiresAt');
+ if (sibling.mode !== 'continue') throw new Error('Expected sibling');
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods,
+ getGenerationJob: async () => null,
+ });
+ await expect(
+ resolve(sibling, {
+ idempotencyKey: getAgentTriggerIdempotencyKey(sibling),
+ requiredWorkerCapability: capability,
+ }),
+ ).resolves.toEqual({ status: 'settled' });
+ expect(
+ (await methods.getAgentBackgroundToolResultBatch(owner(key)))?.proofCopiedAt,
+ ).toBeInstanceOf(Date);
+ expect(await Delivery.findOne({ deliveryKey: key }).lean()).toHaveProperty('expiresAt');
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'one' }),
+ ).toMatchObject({ appliedAt: expect.any(Date) });
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'two' }),
+ ).toMatchObject({ appliedAt: expect.any(Date) });
+});
+
+it.each([
+ ['local', false],
+ ['reconstructed', false],
+ ['local', true],
+ ['reconstructed', true],
+] as const)(
+ 'recovers a released physical projection through a %s poll after successor retirement %s',
+ async (pollPath, replaceOwner) => {
+ const root = await ready('stale-task');
+ const key = getAgentTriggerIdempotencyKey(root);
+ const scope = owner(key);
+ await project('stale-task');
+ const old = await methods.claimAgentBackgroundToolResultBatch({
+ ...scope,
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ if (old.status !== 'acquired') throw new Error('Expected predecessor batch');
+ let pause: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const paused = new Promise<void>((resolve) => {
+ pause = resolve;
+ });
+ const barrier = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ const Message = mongoose.models.Message;
+ const update = Message.findOneAndUpdate.bind(Message);
+ const lateProjection = jest
+ .spyOn(Message, 'findOneAndUpdate')
+ .mockImplementationOnce((...args) => {
+ const query = update(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ pause();
+ await barrier;
+ return execute();
+ });
+ return query;
+ });
+ const delayed = methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'stale-task',
+ kind: 'wakeup',
+ claimId: key,
+ batchId: old.batchId,
+ limit: 1,
+ });
+ await paused;
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...scope,
+ claimId: key,
+ batchId: old.batchId,
+ }),
+ ).toBe(true);
+ if (replaceOwner) {
+ const successor = await methods.claimAgentBackgroundToolResultBatch({
+ ...scope,
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ if (successor.status !== 'acquired') throw new Error('Expected successor batch');
+ expect(successor.batchId).not.toBe(old.batchId);
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...scope,
+ claimId: key,
+ batchId: successor.batchId,
+ }),
+ ).toBe(true);
+ }
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: key,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ reason: 'unadmitted owner retired',
+ settledAt: new Date(),
+ onlyIfUnclaimed: true,
+ }),
+ ).toBe(true);
+ resume();
+ await expect(delayed).resolves.toMatchObject({ status: 'acquired' });
+ lateProjection.mockRestore();
+ expect(await methods.getAgentBackgroundToolResultBatch(scope)).toBeNull();
+
+ // A broad logical-owner release would also erase these unrelated claims.
+ await project('other-epoch');
+ await project('manual-task');
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'other-epoch',
+ kind: 'wakeup',
+ claimId: key,
+ batchId: 'unrelated-epoch',
+ limit: 1,
+ });
+ await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'manual-task',
+ kind: 'manual',
+ claimId: key,
+ });
+ if (pollPath === 'local') {
+ const created = backgroundTaskRegistry.create({
+ taskId: 'stale-task',
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ toolCallId: 'stale-task',
+ toolName: 'tool',
+ });
+ if ('atCapacity' in created) throw new Error('Unexpected capacity');
+ backgroundTaskRegistry.complete(userId, conversationId, created.task.id, {
+ content: 'durable-stale-task',
+ });
+ backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, created.task.id);
+ }
+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ try {
+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ async (deliveryKey, sourceId, reason, options) =>
+ methods.retireAgentTriggerDelivery({
+ deliveryKey,
+ sourceId,
+ reason,
+ settledAt: new Date(),
+ ...options,
+ }),
+ release,
+ async () => null,
+ ({ userId, conversationId, claimId }) =>
+ manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId),
+ methods.releaseAgentBackgroundToolResultClaims,
+ methods,
+ (...args) => manager.getGenerationAdmissionEvidence(...args),
+ );
+ const recoverDeadBackgroundToolClaim = jest.fn(recover);
+ const polled = JSON.parse(
+ await runCheckBackgroundTask({
+ userId,
+ conversationId,
+ args: { background_task_id: 'stale-task' },
+ toolCallId: 'stale-poll',
+ runId: 'recovery-run',
+ generationId: 'recovery-generation',
+ claimBackgroundToolResult: (input) =>
+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),
+ recoverDeadBackgroundToolClaim,
+ }),
+ );
+ expect(polled).toMatchObject({ status: 'completed', result: 'durable-stale-task' });
+ expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith(
+ expect.objectContaining({ claimId: key, batchId: old.batchId }),
+ );
+ expect(release).toHaveBeenCalledWith(
+ expect.objectContaining({ kind: 'wakeup', claimId: key, batchId: old.batchId }),
+ );
+ for (const [taskId, claim] of [
+ ['other-epoch', { kind: 'wakeup', claimId: key, batchId: 'unrelated-epoch' }],
+ ['manual-task', { kind: 'manual', claimId: key, receiptReconciled: true }],
+ ] as const) {
+ expect(
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId,
+ kind: 'wakeup',
+ claimId: 'probe',
+ }),
+ ).toMatchObject({ status: 'claimed', claim });
+ }
+ } finally {
+ await manager.destroy();
+ }
+ },
+);
+
+it('does not release a stale epoch while a successor receipt batch owns the logical delivery', async () => {
+ const root = await ready('stale-task');
+ const key = getAgentTriggerIdempotencyKey(root);
+ const scope = owner(key);
+ await project('stale-task');
+ const old = await methods.claimAgentBackgroundToolResultBatch({
+ ...scope,
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ if (old.status !== 'acquired') throw new Error('Expected predecessor');
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...scope,
+ claimId: key,
+ batchId: old.batchId,
+ });
+ const successor = await methods.claimAgentBackgroundToolResultBatch({
+ ...scope,
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ if (successor.status !== 'acquired') throw new Error('Expected successor');
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'stale-task',
+ kind: 'wakeup',
+ claimId: key,
+ batchId: successor.batchId,
+ limit: 1,
+ });
+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);
+ const retire = jest.fn(async () => true);
+ const fence = jest.fn(async () => 'fenced' as const);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ async () => null,
+ fence,
+ methods.releaseAgentBackgroundToolResultClaims,
+ methods,
+ );
+ expect(
+ await recover({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ claimId: key,
+ batchId: old.batchId,
+ }),
+ ).toBe(false);
+ expect(retire).not.toHaveBeenCalled();
+ expect(fence).not.toHaveBeenCalled();
+ expect(release).not.toHaveBeenCalled();
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...scope, taskId: 'stale-task' }),
+ ).toMatchObject({ batchId: successor.batchId });
+ expect(
+ await methods.claimBackgroundToolResults({
+ userId,
+ conversationId,
+ messageId: parentMessageId,
+ taskId: 'stale-task',
+ kind: 'manual',
+ claimId: 'probe',
+ }),
+ ).toMatchObject({ status: 'claimed', claim: { claimId: key, batchId: successor.batchId } });
+});
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
index 3d8c9460a1..f599dad7a6 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
@@ -1,4 +1,7 @@
-import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
+import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+} from '@librechat/data-schemas';
import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
import type { CodeApprovalMode } from 'librechat-data-provider';
import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
@@ -113,6 +116,37 @@ describe('background tool completion wakeups', () => {
jest.useRealTimers();
});
+ it('admits v3 receipts only when the deployment rollout gate is enabled', async () => {
+ const enqueue = jest.fn<
+ ReturnType<EnqueueBackgroundToolCompletion>,
+ Parameters<EnqueueBackgroundToolCompletion>
+ >(async () => ({ deliveryKey: 'key' }));
+ let enabled = false;
+ const notify = createBackgroundToolCompletionWakeupHandler(
+ enqueue,
+ async () => true,
+ async () => true,
+ undefined,
+ undefined,
+ () => enabled,
+ );
+ await notify(registration());
+ expect(enqueue).toHaveBeenLastCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ }),
+ );
+ enabled = true;
+ await notify(registration({ taskId: 'task-2' }));
+ expect(enqueue).toHaveBeenLastCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+ }),
+ );
+ });
+
it('expedites its own delivery when a result it can consume appears', async () => {
const expedite = jest.fn();
const notify = createBackgroundToolCompletionWakeupHandler(
@@ -265,6 +299,7 @@ describe('background tool completion wakeups', () => {
userId: 'user-1',
conversationId: 'conversation-1',
messageId: 'response-1',
+ allowMissingMessage: true,
kind: 'wakeup',
claimId: 'batch-root-delivery',
});
@@ -274,6 +309,7 @@ describe('background tool completion wakeups', () => {
conversationId: 'conversation-1',
parentMessageId: 'response-1',
claimId: 'batch-root-delivery',
+ recoveryFenced: true,
});
});
@@ -1034,3 +1070,204 @@ describe('pending background completions', () => {
}
});
});
+
+describe('capability-gated receipt batch resolution', () => {
+ function batchMethods() {
+ const { methods } = resolverMethods();
+ return {
+ ...methods,
+ claimAgentBackgroundToolResultBatch: jest.fn(async () => ({
+ status: 'acquired' as const,
+ batchId: 'batch-1',
+ results: [
+ {
+ taskId: 'task-1',
+ toolCallId: 'call-1',
+ toolName: 'tool',
+ status: 'completed' as const,
+ output: 'one',
+ },
+ {
+ taskId: 'task-2',
+ toolCallId: 'call-2',
+ toolName: 'tool',
+ status: 'completed' as const,
+ output: 'two',
+ },
+ ],
+ })),
+ beginAgentBackgroundToolResultBatchDispatch: jest.fn(async () => true),
+ confirmAgentBackgroundToolResultBatch: jest.fn(async () => true),
+ };
+ }
+
+ it('does not probe batch storage for rollout-default v2 deliveries', async () => {
+ const methods = batchMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ await resolve(await envelope(), {
+ idempotencyKey: 'delivery-1',
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ });
+ expect(methods.claimAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
+ });
+
+ it('reconciles every member before dispatch and confirms only after admission', async () => {
+ const methods = batchMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
+ expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ taskId: 'task-1' }),
+ );
+ expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ taskId: 'task-2' }),
+ );
+ expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
+ expect(methods.claimAgentBackgroundToolResultBatch).toHaveBeenCalledWith(
+ expect.objectContaining({
+ tenantId: 'tenant-1',
+ limit: 8,
+ maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
+ }),
+ );
+ if (prepared?.status !== 'ready') throw new Error('Expected ready');
+ expect(prepared.input).toContain('task-2');
+ await prepared.settleOnAdmission?.({
+ mode: 'continue',
+ status: 'started',
+ conversationId: 'conversation-1',
+ });
+ expect(methods.confirmAgentBackgroundToolResultBatch).toHaveBeenCalledTimes(1);
+ });
+
+ it('defers followers until the root has confirmed admission', async () => {
+ const methods = batchMethods();
+ methods.claimAgentBackgroundToolResultBatch
+ .mockResolvedValueOnce({
+ status: 'claimed',
+ claimId: 'root',
+ ownerStatus: 'pending',
+ } as never)
+ .mockResolvedValueOnce({
+ status: 'claimed',
+ claimId: 'root',
+ ownerStatus: 'applied',
+ } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({
+ code: 'BACKGROUND_TOOL_BATCH_PENDING',
+ deferWithoutAttempt: true,
+ });
+ await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).resolves.toEqual({
+ status: 'settled',
+ });
+ expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
+ });
+
+ it('uses fenced dead-owner recovery before retrying a follower as its own root', async () => {
+ const methods = batchMethods();
+ const recoverDeadClaim = jest.fn(async () => true);
+ methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({
+ status: 'claimed',
+ claimId: 'root',
+ ownerStatus: 'recoverable',
+ } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ recoverDeadClaim,
+ });
+ await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({
+ code: 'BACKGROUND_TOOL_BATCH_PENDING',
+ });
+ expect(recoverDeadClaim).toHaveBeenCalledWith(
+ expect.objectContaining({ claimId: 'root', messageId: 'response-1' }),
+ );
+ expect((await resolve(await envelope(), { idempotencyKey: 'follower' }))?.status).toBe('ready');
+ });
+
+ it('releases the entire frozen batch when a sibling manual poll wins', async () => {
+ const methods = batchMethods();
+ methods.claimBackgroundToolResults
+ .mockResolvedValueOnce({ status: 'acquired', results: [] })
+ .mockResolvedValueOnce({
+ status: 'claimed',
+ claim: { kind: 'manual', claimId: 'poll' },
+ } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ });
+ expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
+ expect.objectContaining({ claimId: 'root' }),
+ );
+ expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();
+ });
+
+ it('releases receipt ownership even when no message projection has landed', async () => {
+ const methods = batchMethods();
+ methods.claimBackgroundToolResults.mockResolvedValue({ status: 'not_found' } as never);
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });
+ if (prepared?.status !== 'ready') throw new Error('Expected ready');
+ await prepared.releaseOnDefiniteFailure?.();
+ expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();
+ expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalled();
+ });
+
+ it('retains ownership when admission confirmation fails after dispatch', async () => {
+ const methods = batchMethods();
+ methods.confirmAgentBackgroundToolResultBatch.mockRejectedValueOnce(new Error('lost reply'));
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });
+ if (prepared?.status !== 'ready') throw new Error('Expected ready');
+ await expect(
+ prepared.settleOnAdmission?.({
+ mode: 'continue',
+ status: 'started',
+ conversationId: 'conversation-1',
+ }),
+ ).rejects.toThrow('lost reply');
+ expect(methods.releaseAgentBackgroundToolResultClaims).not.toHaveBeenCalled();
+ });
+
+ it('still detects a lost producer while a v3 receipt is not ready', async () => {
+ const methods = batchMethods();
+ methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
+ methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({
+ status: 'not_ready',
+ waitingForResult: true,
+ } as never);
+ methods.getAgentTriggerDeliveryProducerLease.mockResolvedValueOnce({
+ status: 'expired',
+ leaseUntil: new Date(0),
+ });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({
+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',
+ retryable: false,
+ });
+ });
+});
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts
index e3df97eb04..56a057f95d 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts
@@ -1,6 +1,10 @@
import { randomUUID } from 'node:crypto';
import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider';
-import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
+import {
+ AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+} from '@librechat/data-schemas';
import type {
AgentTriggerProducerLeaseStatus,
AgentTriggerDeliveryMethods,
@@ -80,6 +84,10 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
output: string;
settledAt: Date;
} | null>;
+ persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
+ beginAgentBackgroundToolResultBatchDispatch?: AgentTriggerDeliveryMethods['beginAgentBackgroundToolResultBatchDispatch'];
+ claimAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResultBatch'];
+ confirmAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['confirmAgentBackgroundToolResultBatch'];
claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
@@ -98,6 +106,13 @@ export interface BackgroundToolCompletionWakeupResolverDeps {
methods: WakeupMethods;
getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
getResultBatchSize?: () => number | undefined;
+ recoverDeadClaim?: BackgroundToolDeadClaimRecovery;
+ getGenerationAdmissionEvidence?: (
+ userId: string,
+ clientRequestId: string,
+ streamId: string,
+ conversationId?: string,
+ ) => Promise<{ generationId: string; generationCreatedAt: number } | null>;
/** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */
getWaitMaxIntervalMs?: () => number | undefined;
}
@@ -265,12 +280,14 @@ export function createBackgroundToolCompletionWakeupResolver({
getGenerationJob,
getResultBatchSize,
getWaitMaxIntervalMs,
+ recoverDeadClaim,
+ getGenerationAdmissionEvidence,
}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
AgentTriggerExecutionHostDeps['prepareContinue']
> {
const waitingRetry = (receivedAt: number): string =>
waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);
- return async (
+ const resolve = async (
envelope: AgentContinueTriggerEnvelope,
context: AgentTriggerDispatchContext,
): Promise<AgentTriggerContinuePreparation | undefined> => {
@@ -332,6 +349,313 @@ export function createBackgroundToolCompletionWakeupResolver({
status: 404,
});
}
+ const batchScope = {
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ agentId: envelope.target.agentId,
+ ...(context.deliveryClaimToken != null && { deliveryClaimToken: context.deliveryClaimToken }),
+ };
+ const batch =
+ context.requiredWorkerCapability != null &&
+ context.requiredWorkerCapability !==
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
+ ? undefined
+ : await methods.claimAgentBackgroundToolResultBatch?.({
+ ...batchScope,
+ limit: getResultBatchSize?.() ?? 8,
+ maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,
+ });
+ if (batch?.status === 'claimed') {
+ if (batch.ownerStatus === 'applied') return { status: 'settled' };
+ if (batch.ownerStatus === 'recoverable') {
+ await recoverDeadClaim?.({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ claimId: batch.claimId,
+ batchId: batch.batchId,
+ });
+ }
+ throw executionError('The background result is owned by another completion delivery.', {
+ code: 'BACKGROUND_TOOL_BATCH_PENDING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+ }
+ if (batch?.status === 'not_ready') {
+ if (batch.waitingForResult === true) {
+ // A producer can fail its receipt write while its terminal message
+ // projection succeeds. Reconstruct from that already-durable result.
+ const projection = await methods.claimBackgroundToolResults({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskId: registration.taskId,
+ agentId: envelope.target.agentId,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ limit: 1,
+ });
+ if (
+ projection.status === 'claimed' &&
+ projection.claim?.kind === 'manual' &&
+ projection.claim.receiptReconciled === true
+ )
+ return { status: 'settled' };
+ if (projection.status === 'acquired') {
+ const result = projection.results.find((result) => result.taskId === registration.taskId);
+ const releaseProjection = async (): Promise<void> => {
+ if (
+ !(await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskIds: [registration.taskId],
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ allowMissingMessage: true,
+ }))
+ ) {
+ throw new Error('Background projection reconstruction release was not confirmed');
+ }
+ };
+ let restored = false;
+ try {
+ if (result != null)
+ restored =
+ (await methods.persistAgentBackgroundToolResult?.({
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ result: {
+ status: result.status,
+ output: truncateMiddle(
+ result.output,
+ AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
+ ),
+ settledAt: result.settledAt ?? new Date(envelope.event.occurredAt),
+ },
+ })) === true;
+ } finally {
+ await releaseProjection();
+ }
+ if (restored) return resolve(envelope, context);
+ throw executionError('The background result receipt is being reconstructed.', {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+ }
+ if (
+ projection.status === 'claimed' &&
+ projection.claim?.kind === 'manual' &&
+ projection.claim.generationId != null
+ ) {
+ await recoverDeadClaim?.({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ claimId: projection.claim.claimId,
+ kind: 'manual',
+ generationId: projection.claim.generationId,
+ onlyIfUnreconciled: true,
+ });
+ }
+ if (projection.status === 'claimed')
+ throw executionError('The background projection is being reconciled.', {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+
+ const producer = await methods.getAgentTriggerDeliveryProducerLease({
+ deliveryKey: context.idempotencyKey,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ now: new Date(),
+ });
+ if (producer.status === 'expired') {
+ throw executionError('The process-local background tool executor was lost.', {
+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',
+ retryable: false,
+ });
+ }
+ }
+ throw executionError('The background result batch is not durable yet.', {
+ code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: waitingRetry(envelope.receivedAt),
+ });
+ }
+ if (batch?.status === 'acquired') {
+ const owned = { ...batchScope, batchId: batch.batchId };
+ const dispatchId = randomUUID();
+ let dispatchBegan = false;
+ const release = async (): Promise<boolean> => {
+ const admitted = await getGenerationAdmissionEvidence?.(
+ userId,
+ context.idempotencyKey,
+ envelope.target.conversationId,
+ envelope.target.conversationId,
+ );
+ if (admitted != null) {
+ await methods.confirmAgentBackgroundToolResultBatch?.(owned);
+ return false;
+ }
+ // Absence is not proof. The storage CAS permits only a first definite
+ // failure or a never-dispatched plan, never a retry of an ambiguous POST.
+ const released = await methods.releaseAgentBackgroundToolResultClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ claimId: context.idempotencyKey,
+ batchId: batch.batchId,
+ ...(dispatchBegan && { dispatchId }),
+ ...(context.deliveryClaimToken != null && {
+ deliveryClaimToken: context.deliveryClaimToken,
+ }),
+ });
+ return released === true;
+ };
+ // A receipt may precede its projection. Reconcile every member, not just
+ // the root, against a concurrent manual poll or late message persistence.
+ const projections = await Promise.all(
+ batch.results.map((result) =>
+ methods.claimBackgroundToolResults({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskId: result.taskId,
+ agentId: envelope.target.agentId,
+ kind: 'wakeup',
+ claimId: context.idempotencyKey,
+ batchId: batch.batchId,
+ limit: 1,
+ }),
+ ),
+ );
+ if (
+ projections.some(
+ (projection) =>
+ projection.status === 'claimed' || projection.status === 'outcome_unknown',
+ )
+ ) {
+ for (let index = 0; index < projections.length; index++) {
+ const stale = projections[index];
+ if (stale?.status !== 'claimed' || stale.claim?.kind !== 'wakeup') continue;
+ const taskId = batch.results[index]!.taskId;
+ const receipt = await methods.getAgentBackgroundToolResultClaim?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId: envelope.target.conversationId,
+ parentMessageId: envelope.target.parentMessageId,
+ taskId,
+ });
+ if (receipt?.claimId !== context.idempotencyKey || receipt.batchId !== batch.batchId)
+ continue;
+ await methods.releaseBackgroundToolResultClaims({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ taskIds: [taskId],
+ kind: 'wakeup',
+ claimId: stale.claim.claimId,
+ ...(stale.claim.batchId != null && { batchId: stale.claim.batchId }),
+ allowMissingMessage: true,
+ });
+ throw executionError('A stale background projection is being reconciled.', {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+ }
+ const released = await release();
+ const rootProjection =
+ projections[batch.results.findIndex((result) => result.taskId === registration.taskId)];
+ if (
+ released &&
+ rootProjection?.status === 'claimed' &&
+ rootProjection.claim?.kind === 'manual' &&
+ rootProjection.claim.receiptReconciled === true
+ ) {
+ return { status: 'settled' };
+ }
+ if (released && recoverDeadClaim != null) {
+ await Promise.all(
+ projections.map(async (projection) => {
+ if (
+ projection.status !== 'claimed' ||
+ projection.claim?.kind !== 'manual' ||
+ projection.claim.receiptReconciled === true ||
+ projection.claim.generationId == null
+ )
+ return;
+ await recoverDeadClaim({
+ userId,
+ conversationId: envelope.target.conversationId,
+ messageId: envelope.target.parentMessageId,
+ claimId: projection.claim.claimId,
+ kind: 'manual',
+ generationId: projection.claim.generationId,
+ onlyIfUnreconciled: true,
+ });
+ }),
+ );
+ }
+ throw executionError('Background result ownership is being reconciled.', {
+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '1',
+ });
+ }
+ return {
+ status: 'ready',
+ parentMessageId,
+ ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),
+ input: buildWakeupInput(batch.results),
+ releaseOnDefiniteFailure: async () => {
+ await release();
+ },
+ beginDispatch: async () => {
+ dispatchBegan = true;
+ if (
+ !(await methods.beginAgentBackgroundToolResultBatchDispatch?.({ ...owned, dispatchId }))
+ ) {
+ throw executionError('The background batch preparation was superseded.', {
+ code: 'BACKGROUND_TOOL_BATCH_SUPERSEDED',
+ retryable: true,
+ deferWithoutAttempt: true,
+ });
+ }
+ },
+ settleOnAdmission: async (result) => {
+ if (
+ result.status === 'settled' &&
+ getGenerationAdmissionEvidence != null &&
+ (await getGenerationAdmissionEvidence(
+ userId,
+ context.idempotencyKey,
+ envelope.target.conversationId,
+ envelope.target.conversationId,
+ )) == null
+ ) {
+ throw new Error('A recovery fence does not prove background admission');
+ }
+ if (!(await methods.confirmAgentBackgroundToolResultBatch?.(owned))) {
+ throw new Error('Background result batch admission was not confirmed');
+ }
+ },
+ };
+ }
const claim = await methods.claimBackgroundToolResults({
userId,
conversationId: envelope.target.conversationId,
@@ -545,6 +869,7 @@ export function createBackgroundToolCompletionWakeupResolver({
deferWithoutAttempt: true,
});
};
+ return resolve;
}
/** Lists and discards a conversation's undelivered background completions from the
@@ -652,6 +977,7 @@ export function createBackgroundToolCompletionWakeupHandler(
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
persistResult?: PersistBackgroundToolCompletionResult,
expedite?: (deliveryKey: string) => void,
+ getReceiptBatchingEnabled?: () => boolean,
): (
registration: BackgroundToolWakeupRegistration,
) => Promise<BackgroundToolWakeupAdmission | false> {
@@ -695,7 +1021,10 @@ export function createBackgroundToolCompletionWakeupHandler(
availableAt: new Date(
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
),
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability:
+ getReceiptBatchingEnabled?.() === true
+ ? AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3
+ : AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
});
return {
@@ -735,8 +1064,22 @@ export function createBackgroundToolDeadClaimRecovery(
claimId: string;
}) => Promise<'fenced' | 'started' | 'unavailable'>,
releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
+ batchRecovery?: Pick<
+ AgentTriggerDeliveryMethods,
+ 'getAgentBackgroundToolResultBatch' | 'confirmAgentBackgroundToolResultBatch'
+ >,
+ getAdmissionEvidence?: BackgroundToolCompletionWakeupResolverDeps['getGenerationAdmissionEvidence'],
): BackgroundToolDeadClaimRecovery {
- return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
+ return async ({
+ userId,
+ conversationId,
+ messageId,
+ claimId,
+ kind,
+ generationId,
+ batchId,
+ onlyIfUnreconciled,
+ }) => {
if (kind === 'manual') {
if (generationId == null || generationId.length === 0) {
return false;
@@ -753,8 +1096,41 @@ export function createBackgroundToolDeadClaimRecovery(
messageId,
kind: 'manual',
claimId,
+ ...(onlyIfUnreconciled === true && { onlyIfUnreconciled }),
});
}
+ const owner = {
+ deliveryKey: claimId,
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId,
+ parentMessageId: messageId,
+ };
+ const batch = await batchRecovery?.getAgentBackgroundToolResultBatch(owner);
+ if (batchId != null && batch != null && batch.batchId !== batchId) return false;
+ if (batch?.appliedAt != null) {
+ await batchRecovery?.confirmAgentBackgroundToolResultBatch({
+ ...owner,
+ batchId: batch.batchId,
+ });
+ return false;
+ }
+ const confirmAdmission = async (): Promise<boolean> => {
+ const evidence = await getAdmissionEvidence?.(
+ userId,
+ claimId,
+ conversationId,
+ conversationId,
+ );
+ if (evidence == null) return false;
+ if (batch != null)
+ await batchRecovery?.confirmAgentBackgroundToolResultBatch({
+ ...owner,
+ batchId: batch.batchId,
+ });
+ return true;
+ };
+ if (batch != null && (await confirmAdmission())) return false;
const claimGenerationIsActive = async (): Promise<boolean> => {
const generation = await getGenerationJob(conversationId);
return (
@@ -778,30 +1154,39 @@ export function createBackgroundToolDeadClaimRecovery(
* installs a started tombstone that invalidates a delayed creator's token,
* or observes that job creation already won. */
const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId });
- if (generationFence === 'unavailable') {
+ if (generationFence === 'unavailable') return false;
+ if (generationFence === 'started' && batch != null) {
+ // The native claim CAS proves admission even when a custom store cannot
+ // read historical claims or the live generation has been cleaned up.
+ await batchRecovery?.confirmAgentBackgroundToolResultBatch({
+ ...owner,
+ batchId: batch.batchId,
+ });
return false;
}
if (await claimGenerationIsActive()) {
return false;
}
- const released = await releaseClaims({
+ const receiptBatchId = batch?.batchId ?? batchId;
+ const receiptReleased = await releaseReceiptClaims?.({
+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
+ userId,
+ conversationId,
+ parentMessageId: messageId,
+ claimId,
+ ...(receiptBatchId != null && { batchId: receiptBatchId }),
+ recoveryFenced: true,
+ });
+ if (receiptReleased === false) return false;
+ if (batch != null) return receiptReleased === true;
+ return releaseClaims({
userId,
conversationId,
messageId,
+ allowMissingMessage: true,
kind: 'wakeup',
claimId,
+ ...(receiptBatchId != null && { batchId: receiptBatchId }),
});
- if (!released) {
- return false;
- }
- return (
- (await releaseReceiptClaims?.({
- sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
- userId,
- conversationId,
- parentMessageId: messageId,
- claimId,
- })) ?? true
- );
};
}
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index 6eac3eedb1..ce75ddac9f 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -457,6 +457,59 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);
});
+ it('copies committed manual reconciliation into the final background projection retry', async () => {
+ let finishPersistence: (persisted: boolean) => void = () => undefined;
+ let resolveProjection: (() => { resultClaim?: { receiptReconciled?: true } }) | undefined;
+ const tool = makeSearchTool({ calls: 0 });
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: async () => ({
+ renew: async () => true,
+ retire: async () => true,
+ persistResult: async () => true,
+ }),
+ claim: async () => ({ status: 'not_ready' }),
+ persist: (params) => {
+ resolveProjection = params.resolveBackgroundTask;
+ return new Promise<boolean>((resolve) => {
+ finishPersistence = resolve;
+ });
+ },
+ },
+ });
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'manual-marker-dispatch',
+ stepId: 'manual-marker-step',
+ name: tool.name,
+ args: { q: 'marker', run_in_background: true },
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'manual-marker-convo', run_id: 'manual-marker-parent' },
+ });
+ await flushMicrotasks();
+ const taskId = JSON.parse(dispatch.content).background_task_id as string;
+ try {
+ expect(resolveProjection?.().resultClaim).toBeUndefined();
+ expect(
+ backgroundTaskRegistry.claimResult('exec_user', 'manual-marker-convo', taskId, {
+ kind: 'manual',
+ claimId: 'manual-marker-poll',
+ receiptReconciled: true,
+ generationId: 'manual-marker-generation',
+ }),
+ ).toBe('acquired');
+ expect(resolveProjection?.().resultClaim).toMatchObject({ receiptReconciled: true });
+ } finally {
+ finishPersistence(true);
+ await flushMicrotasks();
+ }
+ });
+
it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => {
let finishPersistence: ((persisted: boolean) => void) | undefined;
const persist = jest.fn(
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index 670296a101..bbb1643d7c 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -6446,6 +6446,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
kind: current.resultClaim.kind,
claimId: current.resultClaim.claimId,
claimedAt: new Date(current.resultClaim.claimedAt),
+ ...(current.resultClaim.receiptReconciled === true && {
+ receiptReconciled: true,
+ }),
...(current.resultClaim.generationId == null
? {}
: { generationId: current.resultClaim.generationId }),
diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts
index 63cd918861..38e257df91 100644
--- a/packages/api/src/agents/harvest.ts
+++ b/packages/api/src/agents/harvest.ts
@@ -51,6 +51,7 @@ export interface BackgroundToolResultState {
claimId: string;
claimedAt: Date;
generationId?: string;
+ receiptReconciled?: true;
};
}
diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md
index 5fe25e09aa..3d266ce654 100644
--- a/packages/api/src/agents/triggers/README.md
+++ b/packages/api/src/agents/triggers/README.md
@@ -269,3 +269,75 @@ event whose individual timing or acknowledgment is actionable. `fire`, `steer`,
`continue` deliveries reject the option instead of silently weakening their semantics. Deliveries
with `expectedAction` also reject coalescing because one generation cannot prove several distinct
action fences.
+
+## Background receipt batches
+
+Ordinary background tool and code completions use `background_tool_completion_batch_v3`.
+Older workers cannot claim these rows; v2 receipts retain task-local delivery. Subagent
+completions do not join receipt batches. The existing `completionResultBatchSize` bounds
+selection, with no collection window. Scope is owner, tenant, conversation, launching
+parent message, and target agent. Results launched on different parent messages remain
+separate even when their branches later converge.
+
+The root persists candidate keys before per-result claims, then freezes only successful
+claims in `backgroundToolResultBatch.members`. Retries use that same list. Each member's
+message projection is reconciled before dispatch. A first definite rejection releases the unadmitted plan. Once a handoff may have been
+admitted, later transport failures retain frozen ownership and the same idempotency key. Admission proof is stored before followers settle, and copied to each
+receipt so root retention cannot strand a follower.
+
+Followers defer without consuming attempts. A dead or retired root is recovered through
+the generation-admission fence before claims are released. Losing collectors release
+claims they cannot dispatch. Concurrent roots can still split the ready set; batching is
+opportunistic, not one-turn-per-conversation election.
+
+`triggerDelivery.spec.ts` includes a real-Mongo collecting barrier, crash/lost-reply
+injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress
+harness. Its latency measures receipt admission, not model turns or deployment latency.
+
+Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on
+**every replica**, then enable:
+
+```yaml
+endpoints:
+ agents:
+ backgroundTasks:
+ completionReceiptBatching: true
+```
+
+Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does
+not protect that path. Disabling batching stops new v3 production, not existing
+ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered
+receipts. A successor lease resumes interrupted cleanup before dispatch.
+
+Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup
+matches that identity; its final plan deletion also matches a release identity and the
+queue lease. A durable dispatch counter prevents a definite retry failure from releasing
+an earlier ambiguous handoff. Native recovery fences distinguish unadmitted tombstones
+from actual generation admission. Proven admissions settle followers, never re-present
+results.
+
+Retired owners do not expire while a batch still needs claim release or per-receipt
+admission proof. Cleanup restores the ordinary retention deadline only after every
+member is safe without the owner. Retirement also closes publication and dispatch.
+
+A failed receipt write is repaired from its durable terminal message projection.
+Manual polling marks receipt reconciliation before automatic delivery can settle that
+handoff; speculative manual ownership only defers the automatic delivery.
+
+Manual confirmation resolves a lost reply by reading the exact committed claim.
+Failed confirmation rolls back only unreconciled ownership. If rollback is unavailable,
+automatic delivery invokes manual-generation recovery after that owner is no longer active.
+Its release CAS excludes committed manual handoffs, including confirmation after a stale
+snapshot. Explicit manual recovery keeps its existing generation-fenced contract.
+
+Manual reconciliation survives registry restoration and final response persistence.
+Collecting plans acquire the root before siblings, so empty retirement can clear an
+unowned plan without orphaning claims. Native `started` recovery fences confirm admission
+even when a custom store cannot read historical claims after job cleanup.
+
+Exact durable replays read the current claim without rewriting its handoff proof.
+Applied followers finish interrupted owner proof copying before they settle, including
+retired roots removed from the delivery queue.
+
+Manual polls carry the observed physical batch ID through recovery. A released owner’s
+late projection is cleared by that exact ID; successor epochs and manual claims remain owned.
diff --git a/packages/api/src/agents/triggers/dispatch.ts b/packages/api/src/agents/triggers/dispatch.ts
index 09d7607eca..99fe88e14e 100644
--- a/packages/api/src/agents/triggers/dispatch.ts
+++ b/packages/api/src/agents/triggers/dispatch.ts
@@ -11,6 +11,9 @@ export interface AgentTriggerDispatchContext {
/** Durable delivery attempt metadata, when dispatched by the queue engine. */
attempt?: number;
maxAttempts?: number;
+ /** Exact queue lease that owns this preparation. */
+ deliveryClaimToken?: string;
+ requiredWorkerCapability?: string;
signal?: AbortSignal;
}
@@ -44,7 +47,13 @@ export interface AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerR
export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(
envelope: unknown,
handlers: AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerResult>,
- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
+ options?: {
+ signal?: AbortSignal;
+ attempt?: number;
+ maxAttempts?: number;
+ deliveryClaimToken?: string;
+ requiredWorkerCapability?: string;
+ },
): Promise<ContinueResult | FireResult | SteerResult> {
let normalized: AgentTriggerEnvelope;
try {
@@ -56,6 +65,10 @@ export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(
idempotencyKey: getAgentTriggerIdempotencyKey(normalized),
...(options?.attempt != null && { attempt: options.attempt }),
...(options?.maxAttempts != null && { maxAttempts: options.maxAttempts }),
+ ...(options?.deliveryClaimToken != null && { deliveryClaimToken: options.deliveryClaimToken }),
+ ...(options?.requiredWorkerCapability != null && {
+ requiredWorkerCapability: options.requiredWorkerCapability,
+ }),
...(options?.signal != null && { signal: options.signal }),
};
if (normalized.mode === 'fire') {
diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts
index 697834d4f8..7c151f1026 100644
--- a/packages/api/src/agents/triggers/engine.spec.ts
+++ b/packages/api/src/agents/triggers/engine.spec.ts
@@ -74,7 +74,12 @@ describe('createAgentTriggerDeliveryEngine', () => {
expect(dispatch).toHaveBeenCalledWith(
{ version: 1 },
- { signal: expect.any(AbortSignal), attempt: 1, maxAttempts: 8 },
+ {
+ signal: expect.any(AbortSignal),
+ attempt: 1,
+ maxAttempts: 8,
+ deliveryClaimToken: 'claim-1',
+ },
);
expect(store.beginAttempt).toHaveBeenCalledWith({
id: 'delivery-row-1',
diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts
index e6d6f9e8fc..7f1414ba22 100644
--- a/packages/api/src/agents/triggers/engine.ts
+++ b/packages/api/src/agents/triggers/engine.ts
@@ -92,6 +92,7 @@ export interface AgentTriggerDeliveryRecord {
orderingKey: string;
laneSequence: number;
envelope: unknown;
+ requiredWorkerCapability?: string;
status: AgentTriggerDeliveryStatus;
attempts: number;
availableAt: Date;
@@ -204,7 +205,13 @@ export interface AgentTriggerDeliveryEngineDeps {
store: AgentTriggerDeliveryStore;
dispatch: (
envelope: unknown,
- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
+ options?: {
+ signal?: AbortSignal;
+ attempt?: number;
+ maxAttempts?: number;
+ deliveryClaimToken?: string;
+ requiredWorkerCapability?: string;
+ },
) => Promise<AgentTriggerExecutionResult>;
/** Source-owned terminalization must commit before its delivery can become
* dead, including recovery after a crash that exhausted the attempt budget. */
@@ -489,6 +496,10 @@ export function createAgentTriggerDeliveryEngine(
signal: controller.signal,
attempt,
maxAttempts,
+ deliveryClaimToken: delivery.claimToken,
+ ...(delivery.requiredWorkerCapability != null && {
+ requiredWorkerCapability: delivery.requiredWorkerCapability,
+ }),
});
} catch (error) {
const attemptedAt = now();
diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts
index 6bf30f70b8..30c6eb818e 100644
--- a/packages/api/src/agents/triggers/host.ts
+++ b/packages/api/src/agents/triggers/host.ts
@@ -70,6 +70,8 @@ export type AgentTriggerContinuePreparation =
releaseOnDefiniteFailure?: (error?: AgentTriggerExecutionError) => MaybePromise<void>;
/** Commits the source handoff after generation admission. Failure is
* outcome-ambiguous: the same delivery retries with the same request id. */
+ /** Durable possible-handoff marker written immediately before transport. */
+ beginDispatch?: () => MaybePromise<void>;
settleOnAdmission?: (result: AgentTriggerContinueResult) => MaybePromise<void>;
}
| { status: 'settled' };
@@ -172,7 +174,13 @@ export interface AgentTriggerExecutionHostDeps {
export interface AgentTriggerExecutionHost {
dispatch: (
envelope: unknown,
- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
+ options?: {
+ signal?: AbortSignal;
+ attempt?: number;
+ maxAttempts?: number;
+ deliveryClaimToken?: string;
+ requiredWorkerCapability?: string;
+ },
) => Promise<AgentTriggerExecutionResult>;
}
@@ -636,6 +644,7 @@ async function startRun(
const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl);
const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch;
let response: Response;
+ await readyPreparation?.beginDispatch?.();
try {
response = await fetcher(url, {
method: 'POST',
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index c62b09e8ce..2f0b9fb1d8 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -1,5 +1,6 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@@ -185,6 +186,7 @@ describe('durable agent trigger service', () => {
expect.objectContaining({
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
@@ -211,6 +213,14 @@ describe('durable agent trigger service', () => {
await service.stop();
});
+ it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => {
+ const service = createAgentTriggerService({ mintToken: () => 'test' });
+ expect(service.getBackgroundCompletionReceiptBatching()).toBe(false);
+ await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true });
+ expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);
+ await service.stop();
+ });
+
describe('waiting completion deliveries', () => {
const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };
const flush = () => new Promise((resolve) => setImmediate(resolve));
@@ -407,6 +417,7 @@ describe('durable agent trigger service', () => {
expect.objectContaining({
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index f2786ed860..1dc249251c 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -1,5 +1,6 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@@ -50,6 +51,7 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
export interface AgentTriggerServiceOptions {
completionResultBatchSize?: number;
+ completionReceiptBatching?: boolean;
address?: BoundAddress | string | null;
idlePolling?: {
queuedTurnMaxIntervalMs?: number;
@@ -230,6 +232,7 @@ export interface AgentTriggerService {
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
getBackgroundCompletionResultBatchSize: () => number;
+ getBackgroundCompletionReceiptBatching: () => boolean;
/** Longest a waiting completion delivery re-checks readiness. */
getCompletionWaitMaxIntervalMs: () => number;
/** Best effort: moves waiting completion deliveries forward after what they wait on changed. */
@@ -271,6 +274,7 @@ function createDeliveryStore(
...input,
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
@@ -352,6 +356,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
let boundOrigin: string | undefined;
let backgroundCompletionResultBatchSize = 8;
+ let backgroundCompletionReceiptBatching = false;
let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
let initializePromise: Promise<void> | undefined;
@@ -410,7 +415,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
const dispatchForActivePrincipal = async (
envelope: unknown,
- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },
+ options?: {
+ signal?: AbortSignal;
+ attempt?: number;
+ maxAttempts?: number;
+ deliveryClaimToken?: string;
+ requiredWorkerCapability?: string;
+ },
): Promise<AgentTriggerExecutionResult> => {
const parsed = parseAgentTriggerEnvelope(envelope);
await requireActivePrincipal(parsed.principal.userId);
@@ -619,6 +630,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return {
initialize: (options = {}) => {
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
+ backgroundCompletionReceiptBatching = options.completionReceiptBatching === true;
completionWaitMaxIntervalMs =
options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
@@ -809,6 +821,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return getClaim == null ? null : getClaim(input);
}),
getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,
+ getBackgroundCompletionReceiptBatching: () => backgroundCompletionReceiptBatching,
getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,
expediteCompletionWakeups: (input) => expediteCompletions(input),
releaseBackgroundToolResultClaims: (input) =>
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 40869e850f..3d28faabdc 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -270,7 +270,8 @@ function claimsMirrorExactly(left: TokenIdempotencyClaim, right: TokenIdempotenc
left.claimToken === right.claimToken &&
left.previousClaimToken === right.previousClaimToken &&
left.generationProtocolVersion === right.generationProtocolVersion &&
- left.startedAt === right.startedAt
+ left.startedAt === right.startedAt &&
+ left.recoveryFence === right.recoveryFence
);
}
@@ -3339,7 +3340,7 @@ class GenerationJobManagerClass {
}
const normalized = normalizeTokenClaim(claim, 'admission evidence');
assertClaimMatchesRequest(normalized, streamId, conversationId);
- if (normalized.startedAt == null) {
+ if (normalized.startedAt == null || normalized.recoveryFence === true) {
return null;
}
return {
@@ -3380,14 +3381,14 @@ class GenerationJobManagerClass {
return 'unavailable';
}
if (observed.existing.startedAt != null) {
- return 'started';
+ return observed.existing.recoveryFence === true ? 'fenced' : 'started';
}
let owned = observed;
if (!observed.claimed) {
owned = await this.takeoverGeneration(userId, clientRequestId, streamId, observed.existing);
if (owned.existing?.startedAt != null) {
- return 'started';
+ return owned.existing.recoveryFence === true ? 'fenced' : 'started';
}
if (!owned.claimed || owned.existing == null) {
return 'unavailable';
@@ -3396,7 +3397,7 @@ class GenerationJobManagerClass {
const claim = normalizeTokenClaim(owned.existing, 'background completion recovery fence');
if (claim.startedAt != null) {
- return 'started';
+ return claim.recoveryFence === true ? 'fenced' : 'started';
}
await this.tombstoneObservedGenerationClaim(
userId,
@@ -3405,6 +3406,7 @@ class GenerationJobManagerClass {
claim,
Date.now(),
claim.generationProtocolVersion === 2 ? 2 : 1,
+ true,
);
return 'fenced';
}
@@ -3477,11 +3479,13 @@ class GenerationJobManagerClass {
claim: TokenIdempotencyClaim,
createdAt: number,
generationProtocolVersion: 1 | 2,
+ recoveryFence?: true,
): Promise<TokenIdempotencyClaim> {
const tombstone: TokenIdempotencyClaim = {
...claim,
startedAt: createdAt,
generationProtocolVersion,
+ ...(recoveryFence === true && { recoveryFence }),
};
const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
let markError: unknown;
diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
index 1c8578d86d..117d858e30 100644
--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
@@ -394,6 +394,13 @@ describe('GenerationJobManager start-generation claim', () => {
});
});
+ it('does not mistake a replayed recovery tombstone for actual admission proof', async () => {
+ const args = ['user-1', 'req-recovery-proof', 'stream-proof', 'stream-proof'] as const;
+ expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');
+ expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');
+ expect(await manager.getGenerationAdmissionEvidence(...args)).toBeNull();
+ });
+
it('invalidates an unpublished continuation creator before manual recovery', async () => {
const original = await manager.claimGeneration(
'user-1',
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index 5c705d70bb..467e067985 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -714,6 +714,8 @@ export interface IdempotencyClaimValue {
* already-started/cleaned generation and can never be taken over as an
* abandoned pre-create lease. */
startedAt?: number;
+ /** Recovery closed an unpublished claim without admitting a generation. */
+ recoveryFence?: true;
}
/** Result of an atomic {@link IJobStore.claimIdempotencyKey} attempt. */
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index 14fae0dc87..8c06a6a0dc 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -1063,6 +1063,13 @@ describe('agent background completion batch config', () => {
endpoints: { agents: { backgroundTasks: {} } },
});
expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);
+ expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false);
+ expect(
+ configSchema.parse({
+ version: '1.0',
+ endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } },
+ }).endpoints?.agents?.backgroundTasks?.completionReceiptBatching,
+ ).toBe(true);
for (const completionResultBatchSize of [0, 17, 1.5]) {
expect(
@@ -1171,6 +1178,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
+ completionReceiptBatching: false,
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
@@ -1190,6 +1198,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
+ completionReceiptBatching: false,
completionWakeups: false,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
@@ -1209,6 +1218,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionResultBatchSize: 8,
+ completionReceiptBatching: false,
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: true,
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index f64b03666c..3ea0043fa2 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -1736,8 +1736,9 @@ export const agentsEndpointSchema = baseEndpointSchema
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
.optional()
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
- /** Maximum message-backed sibling results in one continuation.
- * Independent receipts retain task-local delivery ownership. */
+ /** Enable only after every replica has compatible receipt/poll consumers. */
+ completionReceiptBatching: z.boolean().optional().default(false),
+ /** Maximum compatible sibling results in one continuation. */
completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),
/** Cooperative cancellation for process-local ordinary tools. Off
* by default so existing deployments opt into the new control. */
diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts
index e2f7e93ce9..6afad13b80 100644
--- a/packages/data-schemas/src/index.ts
+++ b/packages/data-schemas/src/index.ts
@@ -76,6 +76,7 @@ export {
export {
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index 825fe5a15d..c5417b8d7a 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -347,6 +347,7 @@ export function createMethods(
const agentQueuedTurnMethods = createAgentQueuedTurnMethods(mongoose);
const agentTriggerDeliveryMethods = createAgentTriggerDeliveryMethods(mongoose, {
+ releaseBatchProjections: messageMethods.releaseBackgroundToolResultClaims,
purgeQueuedTurnsForUser: (user) =>
agentQueuedTurnMethods.deleteAllAgentQueuedTurnsForUser({
user: typeof user === 'string' ? new mongoose.Types.ObjectId(user) : user,
diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts
index 5e029e423a..3b87bbf478 100644
--- a/packages/data-schemas/src/methods/message.spec.ts
+++ b/packages/data-schemas/src/methods/message.spec.ts
@@ -5991,3 +5991,94 @@ describe('Message Operations', () => {
});
});
});
+
+describe('receipt-backed claim release before projection', () => {
+ it('allows a missing message only when the receipt owner explicitly opts in', async () => {
+ const input = {
+ userId: new mongoose.Types.ObjectId().toString(),
+ conversationId: 'missing-conversation',
+ messageId: 'missing-parent',
+ kind: 'wakeup' as const,
+ claimId: 'batch-root',
+ };
+ await expect(releaseBackgroundToolResultClaims(input)).resolves.toBe(false);
+ await expect(
+ releaseBackgroundToolResultClaims({ ...input, allowMissingMessage: true }),
+ ).resolves.toBe(true);
+ });
+});
+
+describe('receipt batch projection epochs', () => {
+ it('does not let delayed cleanup remove a successor claim on the same logical request', async () => {
+ await Message.create({
+ messageId: 'epoch-parent',
+ conversationId: 'epoch-conversation',
+ user: 'epoch-owner',
+ isCreatedByUser: false,
+ unfinished: false,
+ content: [
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'epoch-call',
+ output: 'done',
+ backgroundTask: {
+ taskId: 'epoch-task',
+ toolName: 'tool',
+ status: 'completed',
+ completionWakeup: true,
+ completionReceipt: true,
+ },
+ },
+ },
+ ],
+ });
+ const input = {
+ userId: 'epoch-owner',
+ conversationId: 'epoch-conversation',
+ messageId: 'epoch-parent',
+ taskId: 'epoch-task',
+ kind: 'wakeup' as const,
+ claimId: 'logical-root',
+ limit: 1,
+ };
+ expect((await claimBackgroundToolResults({ ...input, batchId: 'old' })).status).toBe(
+ 'acquired',
+ );
+ let pause: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const paused = new Promise<void>((resolve) => {
+ pause = resolve;
+ });
+ const barrier = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ const update = Message.findOneAndUpdate.bind(Message);
+ const oldCleanup = jest.spyOn(Message, 'findOneAndUpdate').mockImplementationOnce((...args) => {
+ const query = update(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ pause();
+ await barrier;
+ return execute();
+ });
+ return query;
+ });
+ const predecessor = releaseBackgroundToolResultClaims({ ...input, batchId: 'old' });
+ await paused;
+ expect(await releaseBackgroundToolResultClaims({ ...input, batchId: 'old' })).toBe(true);
+ expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(
+ 'acquired',
+ );
+ resume();
+ expect(await predecessor).toBe(true);
+ oldCleanup.mockRestore();
+ expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(
+ 'acquired',
+ );
+ expect(await claimBackgroundToolResults({ ...input, batchId: 'old' })).toMatchObject({
+ status: 'claimed',
+ claim: { batchId: 'new' },
+ });
+ });
+});
diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts
index 5d0ec86234..66181b1b7c 100644
--- a/packages/data-schemas/src/methods/message.ts
+++ b/packages/data-schemas/src/methods/message.ts
@@ -567,7 +567,13 @@ export type BackgroundToolResultClaim =
| { status: 'outcome_unknown'; toolName: string }
| {
status: 'claimed';
- claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string };
+ claim?: {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ generationId?: string;
+ batchId?: string;
+ receiptReconciled?: true;
+ };
messageId?: string;
}
| { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string };
@@ -846,6 +852,7 @@ export interface MessageMethods {
claimId: string;
claimedAt: Date;
generationId?: string;
+ receiptReconciled?: true;
};
};
}): Promise<{ matched: boolean; unfinished: boolean }>;
@@ -858,6 +865,8 @@ export interface MessageMethods {
agentId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
+ /** Physical receipt-batch identity, independent of its logical request. */
+ batchId?: string;
/** Response generation that owns this manual result delivery. */
generationId?: string;
/** Manual owner-process takeover after automatic delivery was retired. */
@@ -866,12 +875,24 @@ export interface MessageMethods {
/** Includes JSON escaping, delimiters, and empty result fields. */
maxMetadataChars?: number;
}): Promise<BackgroundToolResultClaim>;
+ confirmBackgroundToolResultClaim(params: {
+ userId: string;
+ conversationId: string;
+ messageId: string;
+ taskId: string;
+ claimId: string;
+ }): Promise<boolean>;
releaseBackgroundToolResultClaims(params: {
userId: string;
conversationId: string;
messageId: string;
/** Omit to release every sibling owned by this exact batch claim. */
taskIds?: string[];
+ /** Receipt-backed results can precede the message projection. */
+ allowMissingMessage?: true;
+ /** Manual rollback must not erase a committed receipt handoff. */
+ onlyIfUnreconciled?: true;
+ batchId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
}): Promise<boolean>;
@@ -1983,7 +2004,15 @@ export function createMessageMethods(
function readBackgroundToolResultClaim(
row: Pick<IMessage, 'content'>,
taskId: string,
- ): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined {
+ ):
+ | {
+ kind: 'manual' | 'wakeup';
+ claimId: string;
+ generationId?: string;
+ batchId?: string;
+ receiptReconciled?: true;
+ }
+ | undefined {
for (const part of row.content ?? []) {
if (part == null || typeof part !== 'object' || Array.isArray(part)) {
continue;
@@ -1993,7 +2022,13 @@ export function createMessageMethods(
tool_call?: {
backgroundTask?: {
taskId?: unknown;
- resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown };
+ resultClaim?: {
+ kind?: unknown;
+ claimId?: unknown;
+ generationId?: unknown;
+ batchId?: unknown;
+ receiptReconciled?: unknown;
+ };
};
};
}
@@ -2010,6 +2045,8 @@ export function createMessageMethods(
return {
kind: claim.kind,
claimId: claim.claimId,
+ ...(typeof claim.batchId === 'string' && { batchId: claim.batchId }),
+ ...(claim.receiptReconciled === true && { receiptReconciled: true }),
...(typeof claim.generationId === 'string' && claim.generationId.length > 0
? { generationId: claim.generationId }
: {}),
@@ -2021,7 +2058,7 @@ export function createMessageMethods(
function parseBackgroundToolResults(
message: IMessage,
- claim: { kind: 'manual' | 'wakeup'; claimId: string },
+ claim: { kind: 'manual' | 'wakeup'; claimId: string; batchId?: string },
): BackgroundToolResultRecord[] {
const results: BackgroundToolResultRecord[] = [];
for (const part of message.content ?? []) {
@@ -2040,7 +2077,7 @@ export function createMessageMethods(
status?: unknown;
cancelled?: unknown;
settledAt?: unknown;
- resultClaim?: { kind?: unknown; claimId?: unknown };
+ resultClaim?: { kind?: unknown; claimId?: unknown; batchId?: unknown };
};
};
};
@@ -2052,7 +2089,8 @@ export function createMessageMethods(
typeof task.toolName !== 'string' ||
(task.status !== 'completed' && task.status !== 'error') ||
task.resultClaim?.kind !== claim.kind ||
- task.resultClaim.claimId !== claim.claimId
+ task.resultClaim.claimId !== claim.claimId ||
+ task.resultClaim.batchId !== claim.batchId
) {
continue;
}
@@ -2133,6 +2171,7 @@ export function createMessageMethods(
kind,
claimId,
generationId,
+ batchId,
allowUnfinished = false,
limit = kind === 'wakeup' ? 8 : 1,
maxMetadataChars,
@@ -2144,6 +2183,7 @@ export function createMessageMethods(
if (
(requestedMessageId != null &&
(requestedMessageId.length === 0 || requestedMessageId.length > 256)) ||
+ (batchId != null && (batchId.length === 0 || batchId.length > 128 || kind !== 'wakeup')) ||
taskId.length === 0 ||
taskId.length > 256 ||
claimId.length === 0 ||
@@ -2214,7 +2254,10 @@ export function createMessageMethods(
}
const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {};
const requestedClaim = readBackgroundToolResultClaim(row, taskId);
- const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;
+ const replaying =
+ requestedClaim?.kind === kind &&
+ requestedClaim.claimId === claimId &&
+ requestedClaim.batchId === batchId;
const candidates: string[] = [];
const costs = new Map<string, number>();
let receiptBacked = false;
@@ -2233,8 +2276,13 @@ export function createMessageMethods(
}
const terminal = task?.status === 'completed' || task?.status === 'error';
const wakeupEligible = kind !== 'wakeup' || task?.completionWakeup === true;
- const resultClaim = task?.resultClaim as { kind?: unknown; claimId?: unknown } | undefined;
- const replay = resultClaim?.kind === kind && resultClaim.claimId === claimId;
+ const resultClaim = task?.resultClaim as
+ | { kind?: unknown; claimId?: unknown; batchId?: unknown }
+ | undefined;
+ const replay =
+ resultClaim?.kind === kind &&
+ resultClaim.claimId === claimId &&
+ resultClaim.batchId === batchId;
const sameAgent =
agentId == null ||
partAgentId == null ||
@@ -2315,105 +2363,103 @@ export function createMessageMethods(
kind,
claimId,
claimedAt,
+ ...(batchId != null && { batchId }),
...(kind === 'manual' && requestedGenerationId != null
? { generationId: requestedGenerationId }
: {}),
};
- const updated = await Message.findOneAndUpdate(
- {
- user: userId,
- conversationId,
- messageId: resolvedMessageId,
- ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
- content: {
- $elemMatch: {
- type: 'tool_call',
- 'tool_call.backgroundTask.taskId': taskId,
- 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
- ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
- ...(replaying
- ? {
- 'tool_call.backgroundTask.resultClaim.kind': kind,
- 'tool_call.backgroundTask.resultClaim.claimId': claimId,
- }
- : /** Missing OR stored null: the in-memory claimable scan, the
- * claim arrayFilters, and the settle stamp all treat a null
- * claim as unclaimed, and the subfield-preserving settle write
- * keeps a persisted null a whole-object rewrite used to drop.
- * `$exists: false` here would strand such a part as terminal
- * but permanently unclaimable. */
- { 'tool_call.backgroundTask.resultClaim': null }),
- },
+ const claimFilter: FilterQuery<IMessage> = {
+ user: userId,
+ conversationId,
+ messageId: resolvedMessageId,
+ ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
+ content: {
+ $elemMatch: {
+ type: 'tool_call',
+ 'tool_call.backgroundTask.taskId': taskId,
+ 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
+ ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),
+ ...(replaying
+ ? {
+ 'tool_call.backgroundTask.resultClaim.kind': kind,
+ 'tool_call.backgroundTask.resultClaim.claimId': claimId,
+ 'tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,
+ }
+ : /** Missing OR stored null: the in-memory claimable scan, the
+ * claim arrayFilters, and the settle stamp all treat a null
+ * claim as unclaimed, and the subfield-preserving settle write
+ * keeps a persisted null a whole-object rewrite used to drop.
+ * `$exists: false` here would strand such a part as terminal
+ * but permanently unclaimable. */
+ { 'tool_call.backgroundTask.resultClaim': null }),
},
- ...(agentId != null
- ? {
- $expr: {
- $anyElementTrue: {
- $map: {
- input: { $ifNull: ['$content', []] },
- as: 'candidate',
- in: {
- $and: [
- { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
- {
- $in: [
- {
- $ifNull: [
- {
- $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
- },
- null,
- ],
- },
- [null, agentId],
- ],
- },
- ],
- },
+ },
+ ...(agentId != null
+ ? {
+ $expr: {
+ $anyElementTrue: {
+ $map: {
+ input: { $ifNull: ['$content', []] },
+ as: 'candidate',
+ in: {
+ $and: [
+ { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },
+ {
+ $in: [
+ {
+ $ifNull: [
+ {
+ $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],
+ },
+ null,
+ ],
+ },
+ [null, agentId],
+ ],
+ },
+ ],
},
},
},
- }
- : {}),
- },
- /** Stamps the claim onto every part this pass admitted. The filtered
- * positional operator selects those parts by predicate, so the write
- * touches only them instead of re-emitting the whole content array, and
- * needs no read-modify-write. Amazon DocumentDB rejects the
- * aggregation-pipeline form this replaces. */
- { $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },
- {
- new: true,
- projection: { content: 1 },
- arrayFilters: [
+ },
+ }
+ : {}),
+ };
+ const updated = replaying
+ ? await Message.findOne(claimFilter).select({ content: 1 }).lean<IMessage | null>()
+ : await Message.findOneAndUpdate(
+ claimFilter,
+ /** Stamps the claim onto every part this pass admitted. The filtered
+ * positional operator selects those parts by predicate, so the write
+ * touches only them instead of re-emitting the whole content array, and
+ * needs no read-modify-write. Amazon DocumentDB rejects the
+ * aggregation-pipeline form this replaces. */
+ { $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },
{
- 'part.type': 'tool_call',
- 'part.tool_call.backgroundTask.taskId': { $in: candidates },
- 'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
- ...(kind === 'wakeup'
- ? { 'part.tool_call.backgroundTask.completionWakeup': true }
- : {}),
- $and: [
+ new: true,
+ projection: { content: 1 },
+ arrayFilters: [
{
- /** Unclaimed, or already held by this exact claimant (replay). */
- $or: [
- { 'part.tool_call.backgroundTask.resultClaim': null },
+ 'part.type': 'tool_call',
+ 'part.tool_call.backgroundTask.taskId': { $in: candidates },
+ 'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },
+ ...(kind === 'wakeup'
+ ? { 'part.tool_call.backgroundTask.completionWakeup': true }
+ : {}),
+ $and: [
{
- 'part.tool_call.backgroundTask.resultClaim.kind': kind,
- 'part.tool_call.backgroundTask.resultClaim.claimId': claimId,
+ 'part.tool_call.backgroundTask.resultClaim': null,
},
+ ...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),
],
},
- ...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),
],
},
- ],
- },
- ).lean<IMessage | null>();
+ ).lean<IMessage | null>();
if (updated == null) {
return { status: 'not_ready' };
}
- const results = parseBackgroundToolResults(updated, { kind, claimId });
+ const results = parseBackgroundToolResults(updated, { kind, claimId, batchId });
const competingClaim = readBackgroundToolResultClaim(updated, taskId);
return results.some((result) => result.taskId === taskId)
? { status: 'acquired', results, ...recoveredSource }
@@ -2424,21 +2470,82 @@ export function createMessageMethods(
};
}
+ /** Manual ownership is committed only after receipt arbitration completes. */
+ async function confirmBackgroundToolResultClaim(
+ input: Parameters<MessageMethods['confirmBackgroundToolResultClaim']>[0],
+ ): Promise<boolean> {
+ const Message = mongoose.models.Message as Model<IMessage>;
+ const identity = {
+ 'tool_call.backgroundTask.taskId': input.taskId,
+ 'tool_call.backgroundTask.resultClaim.kind': 'manual',
+ 'tool_call.backgroundTask.resultClaim.claimId': input.claimId,
+ };
+ try {
+ const confirmed = await Message.updateOne(
+ {
+ user: input.userId,
+ conversationId: input.conversationId,
+ messageId: input.messageId,
+ content: { $elemMatch: identity },
+ },
+ {
+ $set: { 'content.$[part].tool_call.backgroundTask.resultClaim.receiptReconciled': true },
+ },
+ {
+ arrayFilters: [
+ {
+ 'part.tool_call.backgroundTask.taskId': input.taskId,
+ 'part.tool_call.backgroundTask.resultClaim.kind': 'manual',
+ 'part.tool_call.backgroundTask.resultClaim.claimId': input.claimId,
+ },
+ ],
+ },
+ );
+ return confirmed.matchedCount === 1;
+ } catch (error) {
+ // A lost write reply is not an uncommitted handoff. Read only the exact
+ // manual claim, never manufacture success from another claimant.
+ const committed = await Message.exists({
+ user: input.userId,
+ conversationId: input.conversationId,
+ messageId: input.messageId,
+ content: {
+ $elemMatch: {
+ ...identity,
+ 'tool_call.backgroundTask.resultClaim.receiptReconciled': true,
+ },
+ },
+ });
+ if (committed != null) return true;
+ throw error;
+ }
+ }
+
async function releaseBackgroundToolResultClaims({
userId,
conversationId,
messageId,
taskIds,
+ allowMissingMessage,
+ onlyIfUnreconciled,
kind,
claimId,
+ batchId,
}: {
userId: string;
conversationId: string;
messageId: string;
taskIds?: string[];
+ /** Receipt-backed results can precede the message projection. */
+ allowMissingMessage?: true;
+ /** Manual rollback must not erase a committed receipt handoff. */
+ onlyIfUnreconciled?: true;
+ batchId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
}): Promise<boolean> {
+ if (onlyIfUnreconciled === true && kind !== 'manual')
+ throw new TypeError('Unreconciled rollback requires a manual claim');
if (taskIds?.length === 0) {
return true;
}
@@ -2460,6 +2567,10 @@ export function createMessageMethods(
{
'part.tool_call.backgroundTask.resultClaim.kind': kind,
'part.tool_call.backgroundTask.resultClaim.claimId': claimId,
+ 'part.tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,
+ ...(onlyIfUnreconciled === true && {
+ 'part.tool_call.backgroundTask.resultClaim.receiptReconciled': { $ne: true },
+ }),
...(taskIds == null
? {}
: { 'part.tool_call.backgroundTask.taskId': { $in: taskIds } }),
@@ -2474,9 +2585,13 @@ export function createMessageMethods(
messageId,
content: { $not: { $type: 'array' } },
});
- return arraylessRow != null;
+ if (arraylessRow != null) return true;
+ return (
+ allowMissingMessage === true &&
+ (await Message.exists({ user: userId, conversationId, messageId })) == null
+ );
}
- const remaining = parseBackgroundToolResults(updated, { kind, claimId });
+ const remaining = parseBackgroundToolResults(updated, { kind, claimId, batchId });
return taskIds == null
? remaining.length === 0
: !remaining.some((result) => taskIds.includes(result.taskId));
@@ -4249,6 +4364,7 @@ export function createMessageMethods(
updateMessageText,
updateToolCallResult,
claimBackgroundToolResults,
+ confirmBackgroundToolResultClaim,
releaseBackgroundToolResultClaims,
updateMessage,
recordSubagentTaskControlReceipt,
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index c49c46ea22..51e878d37e 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -10,6 +10,7 @@ import type {
import type { IUser } from '~/types/user';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,
@@ -28,6 +29,8 @@ import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequen
import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
import { createConversationModel } from '../models/convo';
+import { createMessageModel } from '../models/message';
+import { createMessageMethods } from './message';
import { createUserModel } from '../models/user';
jest.mock('~/config/winston', () => ({
@@ -56,8 +59,11 @@ beforeAll(async () => {
UserPurge = createAgentTriggerUserPurgeModel(mongoose);
User = createUserModel(mongoose);
createConversationModel(mongoose);
+ createMessageModel(mongoose);
await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);
- methods = createAgentTriggerDeliveryMethods(mongoose);
+ methods = createAgentTriggerDeliveryMethods(mongoose, {
+ releaseBatchProjections: createMessageMethods(mongoose).releaseBackgroundToolResultClaims,
+ });
}, DB_SETUP_TIMEOUT_MS);
afterAll(async () => {
@@ -79,6 +85,14 @@ afterAll(() => {
setAgentEventActorReceiptMetricObserver();
});
+async function confirmBatch(
+ input: Parameters<typeof methods.getAgentBackgroundToolResultBatch>[0],
+) {
+ const batch = await methods.getAgentBackgroundToolResultBatch(input);
+ if (batch == null) return false;
+ return methods.confirmAgentBackgroundToolResultBatch({ ...input, batchId: batch.batchId });
+}
+
function enqueueInput(
overrides: Partial<Parameters<typeof methods.enqueueAgentTriggerDelivery>[0]> = {},
) {
@@ -5554,3 +5568,1079 @@ describe('agent trigger delivery methods', () => {
expect(await UserPurge.countDocuments({ _id: user })).toBe(1);
});
});
+
+describe('background result receipt batches', () => {
+ const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3;
+ const sourceId = 'background-tool-completion';
+ let user: mongoose.Types.ObjectId;
+ beforeEach(() => {
+ user = new mongoose.Types.ObjectId();
+ });
+
+ async function ready(
+ key: string,
+ overrides: Partial<Parameters<typeof methods.enqueueAgentTriggerDelivery>[0]> = {},
+ ) {
+ await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: key,
+ orderingKey: `lane-${key}`,
+ user,
+ tenantId: 'tenant-1',
+ requiredWorkerCapability: capability,
+ envelope: {
+ event: {
+ type: 'background-tool.completion',
+ source: { id: sourceId, type: 'internal' },
+ payload: { taskId: key, toolCallId: `call-${key}`, toolName: 'tool' },
+ },
+ target: { conversationId: 'conversation', parentMessageId: 'parent', agentId: 'agent' },
+ },
+ ...overrides,
+ }),
+ );
+ await methods.persistAgentBackgroundToolResult({
+ deliveryKey: key,
+ sourceId,
+ result: { status: 'completed', output: `output-${key}`, settledAt: START },
+ });
+ }
+
+ function input(deliveryKey: string) {
+ return {
+ deliveryKey,
+ sourceId,
+ userId: user.toString(),
+ tenantId: 'tenant-1',
+ conversationId: 'conversation',
+ parentMessageId: 'parent',
+ agentId: 'agent',
+ limit: 8,
+ maxMetadataChars: 16000,
+ };
+ }
+
+ it('freezes ready membership, defers followers, and confirms every receipt before settlement', async () => {
+ await ready('one');
+ await ready('two');
+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ expect(batch).toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'one' }, { taskId: 'two' }],
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'pending',
+ });
+ await ready('late');
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(batch);
+ expect(await confirmBatch(input('one'))).toBe(true);
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'applied',
+ });
+ await Delivery.deleteOne({ deliveryKey: 'one' });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'applied',
+ });
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('late'))).status).toBe(
+ 'acquired',
+ );
+ });
+
+ it.each(['claim', 'membership'] as const)(
+ 'reconstructs the same candidates after a crash during %s',
+ async (phase) => {
+ await ready('one');
+ await ready('two');
+ const updateOne = Delivery.updateOne.bind(Delivery);
+ const crash =
+ phase === 'claim'
+ ? jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('crash after candidate write');
+ })
+ : jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {
+ const query = updateOne(...args);
+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResultBatch.members'))
+ throw new Error('crash after result claims');
+ return query;
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(
+ 'crash',
+ );
+ crash.mockRestore();
+ await ready('late');
+ const retry = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ expect(retry).toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'one' }, { taskId: 'two' }],
+ });
+ expect(JSON.stringify(retry)).not.toContain('late');
+ },
+ );
+
+ it('reads back frozen membership after a lost write reply', async () => {
+ await ready('one');
+ await ready('two');
+ const updateOne = Delivery.updateOne.bind(Delivery);
+ const lostReply = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {
+ const query = updateOne(...args);
+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResultBatch.members')) {
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ await execute();
+ throw new Error('lost membership reply');
+ });
+ }
+ return query;
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(
+ 'lost membership reply',
+ );
+ lostReply.mockRestore();
+ await ready('late');
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'one' }, { taskId: 'two' }],
+ });
+ });
+
+ it('returns only successful per-result claims when another root wins part of the selection', async () => {
+ await ready('one');
+ await ready('two');
+ await ready('three');
+ const updateMany = Delivery.updateMany.bind(Delivery);
+ const partial = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce((...args) => {
+ return updateMany({ ...args[0], deliveryKey: { $in: ['one', 'three'] } }, args[1], args[2]);
+ });
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'one' }, { taskId: 'three' }],
+ });
+ partial.mockRestore();
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(
+ 'acquired',
+ );
+ });
+
+ it('releases every member and clears the frozen plan after definite rejection', async () => {
+ await ready('one');
+ await ready('two');
+ await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({ ...input('one'), claimId: 'one' }),
+ ).toBe(true);
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),
+ ).toBeNull();
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(
+ 'acquired',
+ );
+ });
+
+ it.each(['dead', 'retired'] as const)(
+ 'keeps followers recoverable when their root is %s',
+ async (terminal) => {
+ await ready('one');
+ await ready('two');
+ await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ await Delivery.updateOne(
+ { deliveryKey: 'one' },
+ terminal === 'dead'
+ ? { $set: { status: 'leased', capabilityStatus: 'dead' } }
+ : { $set: { status: 'succeeded', result: { backgroundToolCompletionRetired: true } } },
+ );
+ await expect(
+ methods.claimAgentBackgroundToolResultBatch(input('two')),
+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'recoverable' });
+ expect(await Delivery.exists({ deliveryKey: 'two' })).not.toBeNull();
+ },
+ );
+
+ it('does not mix tenants, principals, branches, agents, sources, or v2 deliveries', async () => {
+ await ready('one');
+ await ready('v2', {
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ });
+ await ready('tenant', { tenantId: 'tenant-2' });
+ await ready('user', { user: new mongoose.Types.ObjectId() });
+ for (const [key, field, value] of [
+ ['branch', 'parentMessageId', 'other-parent'],
+ ['agent', 'agentId', 'other-agent'],
+ ['conversation', 'conversationId', 'other-conversation'],
+ ] as const) {
+ await ready(key, {
+ envelope: {
+ event: {
+ type: 'background-tool.completion',
+ source: { id: sourceId, type: 'internal' },
+ payload: { taskId: key, toolCallId: key, toolName: 'tool' },
+ },
+ target: {
+ conversationId: 'conversation',
+ parentMessageId: 'parent',
+ agentId: 'agent',
+ [field]: value,
+ },
+ },
+ });
+ }
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'one' }],
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('v2'))).resolves.toEqual({
+ status: 'legacy',
+ });
+ await expect(
+ methods.claimAgentBackgroundToolResultBatch({ ...input('tenant'), tenantId: 'tenant-1' }),
+ ).resolves.toEqual({ status: 'not_ready' });
+ });
+
+ it('prevents v2 workers from acquiring either a root or a follower during a rolling upgrade', async () => {
+ await ready('one');
+ await ready('two');
+ await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'old',
+ claimToken: 'old-token',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
+ }),
+ ).resolves.toBeNull();
+ const next = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'new',
+ claimToken: 'new-token',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60000),
+ workerCapabilities: [capability],
+ });
+ expect(next).not.toBeNull();
+ });
+
+ it('resumes an interrupted release under the successor queue lease', async () => {
+ await ready('one');
+ await ready('two');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'old' } });
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...input('one'),
+ deliveryClaimToken: 'old',
+ });
+ const updateOne = Delivery.findOneAndUpdate.bind(Delivery);
+ const lostReply = jest.spyOn(Delivery, 'findOneAndUpdate').mockImplementationOnce((...args) => {
+ const query = updateOne(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ await execute();
+ throw new Error('lost release fence reply');
+ });
+ return query;
+ });
+ await expect(
+ methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ claimId: 'one',
+ deliveryClaimToken: 'old',
+ }),
+ ).rejects.toThrow('lost release fence reply');
+ lostReply.mockRestore();
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'new' } });
+ expect(
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...input('one'),
+ deliveryClaimToken: 'new',
+ }),
+ ).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }, { taskId: 'two' }] });
+ expect(await confirmBatch(input('one'))).toBe(true);
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'applied',
+ });
+ });
+
+ it('cannot discard a follower after a root acquired its receipt', async () => {
+ await ready('one');
+ await ready('two');
+ await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ await expect(
+ methods.retireAgentTriggerDelivery({
+ deliveryKey: 'two',
+ sourceId,
+ settledAt: START,
+ reason: 'discarded',
+ onlyIfUnclaimed: true,
+ requireTransition: true,
+ }),
+ ).resolves.toBe(false);
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({
+ status: 'acquired',
+ results: [{ taskId: 'one' }, { taskId: 'two' }],
+ });
+ });
+
+ it('keeps rollout-default receipts visible to old manual polling after late projection', async () => {
+ await ready('one', {
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ });
+ await methods.claimAgentBackgroundToolResults({ ...input('one'), claimId: 'one' });
+ const oldLookup = await Delivery.findOne({
+ user,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ 'envelope.target.conversationId': 'conversation',
+ 'envelope.target.parentMessageId': 'parent',
+ 'envelope.event.payload.taskId': 'one',
+ })
+ .select('+backgroundToolResult')
+ .lean();
+ expect(oldLookup?.backgroundToolResult?.resultClaim?.claimId).toBe('one');
+ });
+
+ it('retains frozen membership across an ambiguous POST and a definitely failed retry', async () => {
+ await ready('one');
+ await ready('two');
+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (batch.status !== 'acquired') throw new Error('Expected acquired batch');
+ const owner = { ...input('one'), batchId: batch.batchId };
+ await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'first' });
+ await ready('late');
+ await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'retry' });
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...owner,
+ claimId: 'one',
+ dispatchId: 'retry',
+ }),
+ ).toBe(false);
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(batch);
+ expect(await methods.confirmAgentBackgroundToolResultBatch(owner)).toBe(true);
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input('late'), taskId: 'late' }),
+ ).toBeNull();
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('late'))).status).toBe(
+ 'acquired',
+ );
+ });
+
+ it('releases only a proven first handoff failure', async () => {
+ await ready('one');
+ await ready('two');
+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (batch.status !== 'acquired') throw new Error('Expected acquired batch');
+ const owner = { ...input('one'), batchId: batch.batchId };
+ await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'first' });
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...owner,
+ claimId: 'one',
+ dispatchId: 'different-attempt',
+ }),
+ ).toBe(false);
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...owner,
+ claimId: 'one',
+ dispatchId: 'first',
+ }),
+ ).toBe(true);
+ });
+
+ it.each(['receipts', 'plan'] as const)(
+ 'fences a predecessor already paused before %s cleanup',
+ async (phase) => {
+ await ready('one');
+ await ready('two');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'old' } });
+ const original = await methods.claimAgentBackgroundToolResultBatch({
+ ...input('one'),
+ deliveryClaimToken: 'old',
+ });
+ if (original.status !== 'acquired') throw new Error('Expected batch');
+ let pause: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const paused = new Promise<void>((resolve) => {
+ pause = resolve;
+ });
+ const barrier = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ let intercept = true;
+ const one = Delivery.updateOne.bind(Delivery);
+ const many = Delivery.updateMany.bind(Delivery);
+ const block = (query: ReturnType<typeof Delivery.updateOne>) => {
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ pause();
+ await barrier;
+ return execute();
+ });
+ return query;
+ };
+ const spy =
+ phase === 'plan'
+ ? jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {
+ const query = one(...args);
+ if (
+ intercept &&
+ JSON.stringify(query.getUpdate()).includes('"backgroundToolResultBatch":1')
+ ) {
+ intercept = false;
+ return block(query);
+ }
+ return query;
+ })
+ : jest.spyOn(Delivery, 'updateMany').mockImplementation((...args) => {
+ const query = many(...args);
+ if (
+ intercept &&
+ JSON.stringify(query.getUpdate()).includes('"backgroundToolResult.resultClaim":1')
+ ) {
+ intercept = false;
+ return block(query);
+ }
+ return query;
+ });
+ const stale = methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ batchId: original.batchId,
+ claimId: 'one',
+ deliveryClaimToken: 'old',
+ });
+ await paused;
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'new' } });
+ const successor = await methods.claimAgentBackgroundToolResultBatch({
+ ...input('one'),
+ deliveryClaimToken: 'new',
+ });
+ if (successor.status !== 'acquired') throw new Error('Expected successor');
+ expect(successor.batchId).not.toBe(original.batchId);
+ resume();
+ expect(await stale).toBe(false);
+ spy.mockRestore();
+ expect(
+ await methods.confirmAgentBackgroundToolResultBatch({
+ ...input('one'),
+ batchId: successor.batchId,
+ }),
+ ).toBe(true);
+ await expect(
+ methods.claimAgentBackgroundToolResultBatch(input('two')),
+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
+ },
+ );
+
+ it('does not let a resumed cleanup snapshot erase a replacement batch without a queue token', async () => {
+ await ready('one');
+ await ready('two');
+ const old = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (old.status !== 'acquired') throw new Error('Expected old batch');
+ await Delivery.updateOne(
+ { deliveryKey: 'one' },
+ { $set: { 'backgroundToolResultBatch.releasing': true } },
+ );
+ let pause: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const paused = new Promise<void>((resolve) => {
+ pause = resolve;
+ });
+ const barrier = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ const update = Delivery.findOneAndUpdate.bind(Delivery);
+ const cleanup = jest.spyOn(Delivery, 'findOneAndUpdate').mockImplementationOnce((...args) => {
+ const query = update(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ pause();
+ await barrier;
+ return execute();
+ });
+ return query;
+ });
+ const predecessor = methods.claimAgentBackgroundToolResultBatch(input('one'));
+ await paused;
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ claimId: 'one',
+ batchId: old.batchId,
+ }),
+ ).toBe(true);
+ const current = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (current.status !== 'acquired') throw new Error('Expected replacement batch');
+ resume();
+ expect(await predecessor).toEqual({ status: 'not_ready' });
+ cleanup.mockRestore();
+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(current);
+ });
+
+ it('clears a collecting-only plan retired by a manual poll after its collector crashes', async () => {
+ await ready('one');
+ await ready('two');
+ const write = Delivery.updateOne.bind(Delivery);
+ const crash = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {
+ const query = write(...args);
+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResult.resultClaim'))
+ throw new Error('crash before root claim');
+ return query;
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(
+ 'crash before root claim',
+ );
+ crash.mockRestore();
+ const collecting = await Delivery.findOne({ deliveryKey: 'one' })
+ .select('+backgroundToolResultBatch +backgroundToolResult')
+ .lean();
+ expect(collecting?.backgroundToolResultBatch).toMatchObject({ candidates: ['one', 'two'] });
+ expect(collecting?.backgroundToolResult?.resultClaim).toBeUndefined();
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ settledAt: START,
+ reason: 'manual poll',
+ onlyIfUnclaimed: true,
+ requireTransition: true,
+ }),
+ ).toBe(true);
+ const retired = await Delivery.findOne({ deliveryKey: 'one' })
+ .select('+backgroundToolResultBatch')
+ .lean();
+ expect(retired).toMatchObject({
+ status: 'succeeded',
+ expiresAt: new Date(START.getTime() + 90 * 24 * 60 * 60_000),
+ });
+ expect(retired).not.toHaveProperty('backgroundToolResultBatch');
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),
+ ).toBeNull();
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(
+ 'acquired',
+ );
+ });
+
+ it('prevents a stale collector from acquiring siblings after empty-plan retirement', async () => {
+ await ready('one');
+ await ready('two');
+ let pause: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const paused = new Promise<void>((resolve) => {
+ pause = resolve;
+ });
+ const barrier = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ const write = Delivery.updateOne.bind(Delivery);
+ const rootClaim = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {
+ const query = write(...args);
+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResult.resultClaim')) {
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ pause();
+ await barrier;
+ return execute();
+ });
+ }
+ return query;
+ });
+ const collecting = methods.claimAgentBackgroundToolResultBatch(input('one'));
+ await paused;
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ settledAt: START,
+ reason: 'manual poll',
+ onlyIfUnclaimed: true,
+ }),
+ ).toBe(true);
+ resume();
+ expect(await collecting).toEqual({ status: 'not_ready' });
+ rootClaim.mockRestore();
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),
+ ).toBeNull();
+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt');
+ });
+
+ it('retains an owned collecting plan before membership publication', async () => {
+ await ready('one');
+ await ready('two');
+ const crash = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('crash before sibling claims');
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(
+ 'crash before sibling claims',
+ );
+ crash.mockRestore();
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ settledAt: START,
+ reason: 'manual poll',
+ onlyIfUnclaimed: true,
+ }),
+ ).toBe(false);
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ settledAt: START,
+ reason: 'recovery',
+ onlyIfDead: true,
+ }),
+ ).toBe(true);
+ const retired = await Delivery.findOne({ deliveryKey: 'one' })
+ .select('+backgroundToolResultBatch')
+ .lean();
+ expect(retired?.backgroundToolResultBatch).toBeDefined();
+ expect(retired).not.toHaveProperty('expiresAt');
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ claimId: 'one',
+ recoveryFenced: true,
+ }),
+ ).toBe(true);
+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt');
+ });
+
+ it('retains a retired owner across the TTL horizon until every claim is released', async () => {
+ await ready('one');
+ await ready('two');
+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (batch.status !== 'acquired') throw new Error('Expected batch');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ reason: 'recovery',
+ onlyIfDead: true,
+ settledAt: START,
+ }),
+ ).toBe(true);
+ // The recovering process died before the generation fence or result release.
+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt');
+ const future = new Date(Date.now() + 91 * 24 * 60 * 60_000);
+ await Delivery.deleteMany({ expiresAt: { $lte: future } });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'recoverable',
+ });
+ expect(
+ await methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ claimId: 'one',
+ batchId: batch.batchId,
+ recoveryFenced: true,
+ }),
+ ).toBe(true);
+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt');
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(
+ 'acquired',
+ );
+ });
+
+ it('retains applied ownership until every receipt has durable proof of admission', async () => {
+ await ready('one');
+ await ready('two');
+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (batch.status !== 'acquired') throw new Error('Expected batch');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ reason: 'recovery',
+ onlyIfDead: true,
+ settledAt: START,
+ });
+ const lostCopy = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('crash before receipt proof copy');
+ });
+ await expect(confirmBatch(input('one'))).rejects.toThrow('crash before receipt proof copy');
+ lostCopy.mockRestore();
+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt');
+ await Delivery.deleteMany({
+ expiresAt: { $lte: new Date(Date.now() + 91 * 24 * 60 * 60_000) },
+ });
+ expect(await Delivery.exists({ deliveryKey: 'one' })).not.toBeNull();
+ // The durable root proof remains sufficient after the native key's horizon.
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'applied',
+ });
+ const root = await Delivery.findOne({ deliveryKey: 'one' })
+ .select('+backgroundToolResultBatch')
+ .lean();
+ expect(root?.backgroundToolResultBatch?.proofCopiedAt).toBeInstanceOf(Date);
+ expect(root?.expiresAt).toBeInstanceOf(Date);
+ await Delivery.deleteOne({ deliveryKey: 'one' });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'applied',
+ });
+ });
+
+ it.each(['before-copy', 'partial-copy', 'before-finish'] as const)(
+ 'repairs a retired owner from its applied follower after %s failure',
+ async (phase) => {
+ await ready('one');
+ await ready('two');
+ await ready('three');
+ const acquired = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (acquired.status !== 'acquired') throw new Error('Expected batch');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ reason: 'recovery',
+ onlyIfDead: true,
+ settledAt: START,
+ });
+ const copy = Delivery.updateMany.bind(Delivery);
+ const finish = Delivery.updateOne.bind(Delivery);
+ const failure =
+ phase === 'before-finish'
+ ? jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {
+ const query = finish(...args);
+ if (JSON.stringify(query.getUpdate()).includes('proofCopiedAt'))
+ throw new Error('proof completion failed');
+ return query;
+ })
+ : jest.spyOn(Delivery, 'updateMany').mockImplementationOnce((...args) => {
+ if (phase === 'before-copy') throw new Error('proof copy failed');
+ const query = copy(
+ { ...args[0], deliveryKey: { $in: ['one', 'two'] } },
+ args[1],
+ args[2],
+ );
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ await execute();
+ throw new Error('partial proof copy failed');
+ });
+ return query;
+ });
+ await expect(confirmBatch(input('one'))).rejects.toThrow();
+ failure.mockRestore();
+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt');
+ const cannotRepair = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('retry cannot copy');
+ });
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).rejects.toThrow(
+ 'retry cannot copy',
+ );
+ cannotRepair.mockRestore();
+ await expect(
+ methods.claimAgentBackgroundToolResultBatch(input('two')),
+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
+ const repaired = await Delivery.findOne({ deliveryKey: 'one' })
+ .select('+backgroundToolResultBatch')
+ .lean();
+ expect(repaired?.backgroundToolResultBatch?.proofCopiedAt).toBeInstanceOf(Date);
+ expect(repaired?.expiresAt).toBeInstanceOf(Date);
+ for (const taskId of ['one', 'two', 'three']) {
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input(taskId), taskId }),
+ ).toMatchObject({ appliedAt: expect.any(Date) });
+ }
+ await Delivery.deleteOne({ deliveryKey: 'one' });
+ await expect(
+ methods.claimAgentBackgroundToolResultBatch(input('three')),
+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' });
+ },
+ );
+
+ it('does not publish a late collecting plan onto an already retired owner', async () => {
+ await ready('one');
+ await ready('two');
+ let pause: () => void = () => undefined;
+ let resume: () => void = () => undefined;
+ const paused = new Promise<void>((resolve) => {
+ pause = resolve;
+ });
+ const barrier = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ const update = Delivery.updateOne.bind(Delivery);
+ const publication = jest.spyOn(Delivery, 'updateOne').mockImplementationOnce((...args) => {
+ const query = update(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ pause();
+ await barrier;
+ return execute();
+ });
+ return query;
+ });
+ const stale = methods.claimAgentBackgroundToolResultBatch(input('one'));
+ await paused;
+ expect(
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ reason: 'discarded',
+ onlyIfUnclaimed: true,
+ settledAt: START,
+ }),
+ ).toBe(true);
+ const collectingCrash = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {
+ throw new Error('crash after late publication');
+ });
+ resume();
+ await expect(stale).resolves.toEqual({ status: 'not_ready' });
+ publication.mockRestore();
+ collectingCrash.mockRestore();
+ expect(
+ await Delivery.findOne({ deliveryKey: 'one' }).select('+backgroundToolResultBatch').lean(),
+ ).not.toHaveProperty('backgroundToolResultBatch');
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),
+ ).toBeNull();
+ });
+
+ it('cannot begin dispatch after the owner is retired, even without a queue token', async () => {
+ await ready('one');
+ await ready('two');
+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ if (batch.status !== 'acquired') throw new Error('Expected batch');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });
+ await methods.retireAgentTriggerDelivery({
+ deliveryKey: 'one',
+ sourceId,
+ reason: 'recovery',
+ onlyIfDead: true,
+ settledAt: START,
+ });
+ expect(
+ await methods.beginAgentBackgroundToolResultBatchDispatch({
+ ...input('one'),
+ batchId: batch.batchId,
+ dispatchId: 'late',
+ }),
+ ).toBe(false);
+ });
+
+ it('does not let an expired lease release its successor preparation', async () => {
+ await ready('one');
+ await ready('two');
+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'current' } });
+ expect(
+ (
+ await methods.claimAgentBackgroundToolResultBatch({
+ ...input('one'),
+ deliveryClaimToken: 'current',
+ })
+ ).status,
+ ).toBe('acquired');
+ await expect(
+ methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ claimId: 'one',
+ deliveryClaimToken: 'expired',
+ }),
+ ).resolves.toBe(false);
+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(
+ 'claimed',
+ );
+ expect(await confirmBatch(input('one'))).toBe(true);
+ await expect(
+ methods.releaseAgentBackgroundToolResultClaims({
+ ...input('one'),
+ claimId: 'one',
+ deliveryClaimToken: 'current',
+ }),
+ ).resolves.toBe(false);
+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({
+ status: 'claimed',
+ ownerStatus: 'applied',
+ });
+ });
+
+ it('serializes definite release against an admission confirmation', async () => {
+ await ready('one');
+ await ready('two');
+ await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ const [released, confirmed] = await Promise.all([
+ methods.releaseAgentBackgroundToolResultClaims({ ...input('one'), claimId: 'one' }),
+ confirmBatch(input('one')),
+ ]);
+ expect(Number(released) + Number(confirmed)).toBe(1);
+ });
+
+ it('uses bounded selection, including a batch size of one', async () => {
+ await ready('one');
+ await ready('two');
+ expect(
+ await methods.claimAgentBackgroundToolResultBatch({ ...input('one'), limit: 1 }),
+ ).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });
+ expect(
+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),
+ ).toBeNull();
+ });
+
+ it('keeps private batch state out of delivery reads and erases it with conversation results', async () => {
+ await ready('one');
+ await ready('two');
+ await methods.claimAgentBackgroundToolResultBatch(input('one'));
+ expect(await methods.getAgentTriggerDelivery('one')).not.toHaveProperty(
+ 'backgroundToolResultBatch',
+ );
+ await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation']);
+ const row = await Delivery.findOne({ deliveryKey: 'one' })
+ .select('+backgroundToolResultBatch +backgroundToolResult')
+ .lean();
+ expect(row).not.toHaveProperty('backgroundToolResultBatch');
+ expect(row).not.toHaveProperty('backgroundToolResult');
+ });
+});
+
+describe('background receipt batch contention', () => {
+ async function seed(
+ groups = 1,
+ capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+ ) {
+ const user = new mongoose.Types.ObjectId();
+ const claims: Parameters<typeof methods.claimAgentBackgroundToolResultBatch>[0][] = [];
+ for (let group = 0; group < groups; group++) {
+ for (let task = 0; task < 4; task++) {
+ const key = `group-${group}-task-${task}`;
+ await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: key,
+ orderingKey: key,
+ user,
+ requiredWorkerCapability: capability,
+ envelope: {
+ event: {
+ type: 'background-tool.completion',
+ source: { id: 'background-tool-completion', type: 'internal' },
+ payload: { taskId: key, toolCallId: key, toolName: 'tool' },
+ },
+ target: {
+ conversationId: `conversation-${group}`,
+ parentMessageId: 'parent',
+ agentId: 'agent',
+ },
+ },
+ }),
+ );
+ await methods.persistAgentBackgroundToolResult({
+ deliveryKey: key,
+ sourceId: 'background-tool-completion',
+ result: { status: 'completed', output: key, settledAt: START },
+ });
+ claims.push({
+ deliveryKey: key,
+ sourceId: 'background-tool-completion',
+ userId: user.toString(),
+ tenantId: 'tenant-1',
+ conversationId: `conversation-${group}`,
+ parentMessageId: 'parent',
+ agentId: 'agent',
+ limit: 8,
+ maxMetadataChars: 16000,
+ });
+ }
+ }
+ return claims;
+ }
+
+ it('makes progress when two workers cross the collecting barrier together', async () => {
+ const claims = await seed();
+ let arrivals = 0;
+ let open: () => void = () => undefined;
+ const barrier = new Promise<void>((resolve) => {
+ open = resolve;
+ });
+ const updateMany = Delivery.updateMany.bind(Delivery);
+ const race = jest.spyOn(Delivery, 'updateMany').mockImplementation((...args) => {
+ const query = updateMany(...args);
+ const execute = query.exec.bind(query);
+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {
+ if (++arrivals === 2) open();
+ await barrier;
+ return execute();
+ });
+ return query;
+ });
+ const batches = await Promise.all(
+ claims.slice(0, 2).map((claim) => methods.claimAgentBackgroundToolResultBatch(claim)),
+ );
+ race.mockRestore();
+ const consumed: string[] = [];
+ for (let index = 0; index < batches.length; index++) {
+ const batch = batches[index]!;
+ if (batch.status !== 'acquired') continue;
+ consumed.push(...batch.results.map((result) => result.taskId));
+ await confirmBatch(claims[index]!);
+ }
+ for (const claim of claims) {
+ const batch = await methods.claimAgentBackgroundToolResultBatch(claim);
+ if (batch.status === 'acquired' && !consumed.includes(claim.deliveryKey)) {
+ consumed.push(...batch.results.map((result) => result.taskId));
+ await confirmBatch(claim);
+ }
+ }
+ expect(new Set(consumed).size).toBe(4);
+ expect(consumed).toHaveLength(4);
+ });
+
+ it('reduces 8 conversations by 4 ready receipts without duplicates or stranded results', async () => {
+ const baselineClaims = await seed(
+ 8,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ );
+ let baselineTurns = 0;
+ for (const claim of baselineClaims) {
+ const batch = await methods.claimAgentBackgroundToolResults({
+ ...claim,
+ claimId: claim.deliveryKey,
+ });
+ if (batch.status === 'acquired') baselineTurns++;
+ }
+ await Promise.all([Delivery.deleteMany({}), LaneSequence.deleteMany({})]);
+ const claims = await seed(8);
+ const started = Date.now();
+ const consumed: string[] = [];
+ const latency: number[] = [];
+ let turns = 0;
+ // One ready root per conversation, as with a bounded worker pool behind
+ // a busy parent. A separate barrier test covers same-conversation overlap.
+ for (let round = 0; round < 4; round++) {
+ await Promise.all(
+ Array.from({ length: 8 }, async (_, group) => {
+ const claim = claims[group * 4 + round]!;
+ const batch = await methods.claimAgentBackgroundToolResultBatch(claim);
+ if (batch.status !== 'acquired') return;
+ turns++;
+ consumed.push(...batch.results.map((result) => result.taskId));
+ await confirmBatch(claim);
+ latency.push(Date.now() - started);
+ }),
+ );
+ }
+ const unique = new Set(consumed).size;
+ const report = {
+ uniqueResults: unique,
+ duplicates: consumed.length - unique,
+ strandedResults: 32 - unique,
+ baselineTurns,
+ turns,
+ admissionLatencyMs: Math.max(...latency),
+ };
+ console.info('BACKGROUND_RECEIPT_STRESS', JSON.stringify(report));
+ expect(report).toMatchObject({
+ uniqueResults: 32,
+ duplicates: 0,
+ strandedResults: 0,
+ baselineTurns: 32,
+ turns: 8,
+ });
+ });
+});
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
index 81d4d7140c..22cab0aded 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -1,4 +1,5 @@
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
+import { backgroundResultMetadata } from 'librechat-data-provider';
import type { FilterQuery, Model, Types } from 'mongoose';
import type {
AgentBackgroundToolResultReceipt,
@@ -17,8 +18,10 @@ import type {
IAgentTriggerUserPurge,
IAgentTriggerUserPurgeDocument,
} from '~/types/triggerDelivery';
+import type { MessageMethods } from './message';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
@@ -38,6 +41,10 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
const MAX_PURGE_RECOVERY_LIMIT = 200;
const HISTORY_LIMIT = 64;
const MAX_BATCH_SIZE = 8;
+const BACKGROUND_RECEIPT_CAPABILITIES = [
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+];
const MAX_BATCH_BYTES = 512 * 1024;
/** Bounds the claim's compare-and-swap retries. Each lost round means another
* worker claimed the row this one read, so the queue is making progress. */
@@ -167,6 +174,48 @@ export type AgentBackgroundToolResultClaim =
}>;
};
+export interface AgentBackgroundToolResultBatchInput {
+ deliveryKey: string;
+ sourceId: string;
+ userId: string;
+ tenantId?: string;
+ conversationId: string;
+ parentMessageId: string;
+ agentId: string;
+ limit: number;
+ maxMetadataChars: number;
+ deliveryClaimToken?: string;
+}
+
+export type AgentBackgroundToolResultBatchClaim =
+ | { status: 'legacy' }
+ | { status: 'not_ready'; waitingForResult?: true }
+ | {
+ status: 'claimed';
+ claimId: string;
+ batchId: string;
+ ownerStatus: 'pending' | 'applied' | 'recoverable';
+ }
+ | (Extract<AgentBackgroundToolResultClaim, { status: 'acquired' }> & { batchId: string });
+
+export type AgentBackgroundToolResultBatchOwnerInput = Omit<
+ AgentBackgroundToolResultBatchInput,
+ 'limit' | 'maxMetadataChars' | 'agentId'
+> & { agentId?: string };
+
+export type AgentBackgroundToolResultBatchReleaseInput = {
+ sourceId: string;
+ userId: string;
+ conversationId: string;
+ parentMessageId: string;
+ claimId: string;
+ deliveryClaimToken?: string;
+ batchId?: string;
+ dispatchId?: string;
+ /** Retirement and generation-claim CAS proved that no admission can occur. */
+ recoveryFenced?: true;
+};
+
export interface AgentTriggerDeliveryFence {
id: string;
workerId: string;
@@ -414,13 +463,21 @@ export interface AgentTriggerDeliveryMethods {
claimId: string;
limit?: number;
}) => Promise<AgentBackgroundToolResultClaim>;
- releaseAgentBackgroundToolResultClaims: (input: {
- sourceId: string;
- userId: string;
- conversationId: string;
- parentMessageId: string;
- claimId: string;
- }) => Promise<boolean>;
+ releaseAgentBackgroundToolResultClaims: (
+ input: AgentBackgroundToolResultBatchReleaseInput,
+ ) => Promise<boolean>;
+ claimAgentBackgroundToolResultBatch: (
+ input: AgentBackgroundToolResultBatchInput,
+ ) => Promise<AgentBackgroundToolResultBatchClaim>;
+ getAgentBackgroundToolResultBatch: (
+ input: AgentBackgroundToolResultBatchOwnerInput,
+ ) => Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null>;
+ beginAgentBackgroundToolResultBatchDispatch: (
+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },
+ ) => Promise<boolean>;
+ confirmAgentBackgroundToolResultBatch: (
+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },
+ ) => Promise<boolean>;
settleAgentTriggerHandlingOutcome: (
input: SettleAgentTriggerHandlingOutcomeInput,
) => Promise<boolean>;
@@ -628,6 +685,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliv
export function createAgentTriggerDeliveryMethods(
mongoose: typeof import('mongoose'),
deps: {
+ releaseBatchProjections?: MessageMethods['releaseBackgroundToolResultClaims'];
purgeQueuedTurnsForUser?: (user: string | Types.ObjectId) => Promise<unknown>;
} = {},
): AgentTriggerDeliveryMethods {
@@ -1125,6 +1183,8 @@ export function createAgentTriggerDeliveryMethods(
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
input.requiredWorkerCapability !==
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&
+ input.requiredWorkerCapability !==
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2
@@ -2182,38 +2242,70 @@ export function createAgentTriggerDeliveryMethods(
},
};
})();
- const retired = await Delivery()
- .findOneAndUpdate(
- {
- deliveryKey: input.deliveryKey,
- 'envelope.event.source.type': 'internal',
- 'envelope.event.source.id': input.sourceId,
- ...statusFence,
- },
- {
- $set: {
- status: 'succeeded',
- result,
- settledAt: input.settledAt,
- expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
- laneCleanupPendingAt: input.settledAt,
+ const retire = (retainBatch: boolean, discardEmptyPlan = false) =>
+ Delivery()
+ .findOneAndUpdate(
+ {
+ deliveryKey: input.deliveryKey,
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': input.sourceId,
+ ...statusFence,
+ ...(discardEmptyPlan && {
+ backgroundToolResultBatch: { $exists: true },
+ 'backgroundToolResultBatch.members.0': { $exists: false },
+ 'backgroundToolResult.resultClaim.claimId': { $ne: input.deliveryKey },
+ }),
+ ...(!discardEmptyPlan &&
+ retainBatch && {
+ backgroundToolResultBatch: { $exists: true },
+ 'backgroundToolResultBatch.proofCopiedAt': { $exists: false },
+ }),
+ ...(!discardEmptyPlan &&
+ !retainBatch && {
+ $and: [
+ {
+ $or: [
+ { backgroundToolResultBatch: { $exists: false } },
+ { 'backgroundToolResultBatch.proofCopiedAt': { $exists: true } },
+ ],
+ },
+ ],
+ }),
+ ...(input.onlyIfUnclaimed === true
+ ? { 'backgroundToolResult.resultClaim': { $exists: false } }
+ : {}),
},
- $unset: {
- leaseBy: 1,
- leaseUntil: 1,
- claimToken: 1,
- capabilityStatus: 1,
- claimAvailableAt: 1,
- capabilityLeaseBy: 1,
- capabilityLeaseUntil: 1,
- capabilityClaimToken: 1,
- lastError: 1,
+ {
+ $set: {
+ status: 'succeeded',
+ result,
+ settledAt: input.settledAt,
+ ...(!retainBatch && {
+ expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
+ }),
+ laneCleanupPendingAt: input.settledAt,
+ },
+ $unset: {
+ leaseBy: 1,
+ leaseUntil: 1,
+ claimToken: 1,
+ capabilityStatus: 1,
+ claimAvailableAt: 1,
+ capabilityLeaseBy: 1,
+ capabilityLeaseUntil: 1,
+ capabilityClaimToken: 1,
+ lastError: 1,
+ ...(retainBatch && { expiresAt: 1 }),
+ ...(discardEmptyPlan && { backgroundToolResultBatch: 1 }),
+ },
},
- },
- { new: true },
- )
- .select('_id orderingKey laneCleanupPendingAt')
- .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
+ { new: true },
+ )
+ .select('_id orderingKey laneCleanupPendingAt')
+ .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
+ // A retired owner is its followers' recovery and frozen-membership record.
+ // Its TTL cannot start while release or receipt-proof copying is incomplete.
+ const retired = (await retire(false)) ?? (await retire(false, true)) ?? (await retire(true));
if (retired?._id != null) {
try {
await fulfillLaneCleanupRequest(retired);
@@ -2268,7 +2360,7 @@ export function createAgentTriggerDeliveryMethods(
requiredWorkerCapability: {
$in: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
- AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ ...BACKGROUND_RECEIPT_CAPABILITIES,
],
},
'envelope.event.source.type': 'internal',
@@ -2312,7 +2404,7 @@ export function createAgentTriggerDeliveryMethods(
requiredWorkerCapability: {
$in: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
- AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ ...BACKGROUND_RECEIPT_CAPABILITIES,
],
},
'envelope.event.source.type': 'internal',
@@ -2366,7 +2458,7 @@ export function createAgentTriggerDeliveryMethods(
'envelope.target.conversationId': input.conversationId,
...(input.taskId != null && { 'envelope.event.payload.taskId': input.taskId }),
/** Legacy rows keep results only on the parent message. */
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
};
const read = (filter: FilterQuery<IAgentTriggerDelivery>) =>
Delivery()
@@ -2568,7 +2660,7 @@ export function createAgentTriggerDeliveryMethods(
}
const identity = {
deliveryKey: input.deliveryKey,
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
backgroundToolResultErasedAt: { $exists: false },
@@ -2612,7 +2704,7 @@ export function createAgentTriggerDeliveryMethods(
const delivery = await Delivery()
.findOne({
deliveryKey: input.deliveryKey,
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
})
@@ -2648,7 +2740,7 @@ export function createAgentTriggerDeliveryMethods(
}) {
return {
user: input.userId,
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
'envelope.target.conversationId': input.conversationId,
@@ -2769,38 +2861,592 @@ export function createAgentTriggerDeliveryMethods(
return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };
}
- async function releaseAgentBackgroundToolResultClaims(input: {
- sourceId: string;
- userId: string;
- conversationId: string;
- parentMessageId: string;
- claimId: string;
- }): Promise<boolean> {
+ async function releaseAgentBackgroundToolResultClaims(
+ input: AgentBackgroundToolResultBatchReleaseInput,
+ ): Promise<boolean> {
if (
- input.sourceId.length === 0 ||
- input.sourceId.length > 256 ||
- input.userId.length === 0 ||
- input.conversationId.length === 0 ||
- input.conversationId.length > 256 ||
- input.parentMessageId.length === 0 ||
- input.parentMessageId.length > 256 ||
- input.claimId.length === 0 ||
+ [
+ input.sourceId,
+ input.userId,
+ input.conversationId,
+ input.parentMessageId,
+ input.claimId,
+ ].some((value) => value.length === 0 || value.length > 256) ||
input.claimId.length > 128
) {
throw new TypeError('Invalid background tool result receipt claim release');
}
const scope = backgroundResultIdentity(input);
+ const releaseId = randomUUID();
+ const releasing = await Delivery()
+ .findOneAndUpdate(
+ {
+ ...scope,
+ deliveryKey: input.claimId,
+ backgroundToolResultBatch: { $exists: true },
+ 'backgroundToolResultBatch.appliedAt': { $exists: false },
+ ...(input.batchId == null ? {} : { 'backgroundToolResultBatch.batchId': input.batchId }),
+ ...(input.deliveryClaimToken == null
+ ? {}
+ : { capabilityClaimToken: input.deliveryClaimToken }),
+ ...(input.recoveryFenced === true
+ ? {}
+ : {
+ $or: [
+ { 'backgroundToolResultBatch.releasing': true },
+ { 'backgroundToolResultBatch.dispatchCount': 0 },
+ ...(input.dispatchId == null
+ ? []
+ : [
+ {
+ 'backgroundToolResultBatch.dispatchCount': 1,
+ 'backgroundToolResultBatch.dispatchId': input.dispatchId,
+ },
+ ]),
+ ],
+ }),
+ },
+ {
+ $set: {
+ 'backgroundToolResultBatch.releasing': true,
+ 'backgroundToolResultBatch.releaseId': releaseId,
+ },
+ },
+ { new: true, timestamps: false },
+ )
+ .select('status +backgroundToolResultBatch')
+ .lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();
+ if (
+ releasing == null &&
+ (await Delivery().exists({
+ ...scope,
+ deliveryKey: input.claimId,
+ backgroundToolResultBatch: { $exists: true },
+ })) != null
+ )
+ return false;
+ const batch = releasing?.backgroundToolResultBatch;
+ const batchId = batch?.batchId ?? input.batchId;
+ const receiptScope = {
+ ...scope,
+ 'backgroundToolResult.resultClaim.claimId': input.claimId,
+ 'backgroundToolResult.resultClaim.batchId': batchId ?? null,
+ };
await Delivery().updateMany(
- { ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },
+ { ...receiptScope, 'backgroundToolResult.resultClaim.appliedAt': { $exists: false } },
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
{ timestamps: false },
);
- return (
+ if ((await Delivery().exists(receiptScope)) != null) return false;
+ if (batch != null) {
+ if (deps.releaseBatchProjections == null)
+ throw new Error('Background batch projection cleanup is unavailable');
+ if (
+ !(await deps.releaseBatchProjections({
+ userId: input.userId,
+ conversationId: input.conversationId,
+ messageId: input.parentMessageId,
+ kind: 'wakeup',
+ claimId: input.claimId,
+ batchId: batch.batchId,
+ allowMissingMessage: true,
+ }))
+ )
+ return false;
+ const clear = (retired: boolean) =>
+ Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.claimId,
+ 'backgroundToolResultBatch.batchId': batch.batchId,
+ 'backgroundToolResultBatch.releaseId': releaseId,
+ 'backgroundToolResultBatch.appliedAt': { $exists: false },
+ status: retired ? 'succeeded' : { $ne: 'succeeded' },
+ ...(input.deliveryClaimToken == null
+ ? {}
+ : { capabilityClaimToken: input.deliveryClaimToken }),
+ },
+ {
+ $unset: { backgroundToolResultBatch: 1 },
+ ...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),
+ },
+ { timestamps: false },
+ );
+ const cleared = await clear(releasing?.status === 'succeeded');
+ if (cleared.matchedCount === 1) return true;
+ return releasing?.status !== 'succeeded' && (await clear(true)).matchedCount === 1;
+ }
+ return true;
+ }
+
+ function backgroundBatchScope(
+ input:
+ | AgentBackgroundToolResultBatchInput
+ | Omit<AgentBackgroundToolResultBatchInput, 'limit' | 'maxMetadataChars'>,
+ ) {
+ return {
+ ...backgroundResultIdentity(input),
+ tenantId: input.tenantId ?? null,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+ 'envelope.event.type': 'background-tool.completion',
+ };
+ }
+
+ /** Each result has its own CAS. Candidates survive a collecting crash; only
+ * confirmed owners enter the frozen member list used by every retry. */
+ async function claimAgentBackgroundToolResultBatch(
+ input: AgentBackgroundToolResultBatchInput,
+ ): Promise<AgentBackgroundToolResultBatchClaim> {
+ if (
+ [
+ input.deliveryKey,
+ input.sourceId,
+ input.userId,
+ input.conversationId,
+ input.parentMessageId,
+ input.agentId,
+ ].some((value) => value.length === 0 || value.length > 256) ||
+ input.deliveryKey.length > 128
+ ) {
+ throw new TypeError('Invalid background result batch identity');
+ }
+ if (
+ !Number.isSafeInteger(input.limit) ||
+ input.limit < 1 ||
+ !Number.isSafeInteger(input.maxMetadataChars) ||
+ input.maxMetadataChars < 1
+ ) {
+ throw new TypeError('Invalid background result batch bounds');
+ }
+ const scope = backgroundBatchScope(input);
+ const projection =
+ 'deliveryKey requiredWorkerCapability status capabilityStatus result +backgroundToolResult +backgroundToolResultBatch';
+ type Row = Pick<
+ IAgentTriggerDelivery,
+ | 'deliveryKey'
+ | 'requiredWorkerCapability'
+ | 'status'
+ | 'capabilityStatus'
+ | 'result'
+ | 'backgroundToolResult'
+ | 'backgroundToolResultBatch'
+ > & { envelope?: unknown };
+ const readRoot = () =>
+ Delivery()
+ .findOne({ ...scope, deliveryKey: input.deliveryKey })
+ .select(projection + ' envelope')
+ .lean<Row>();
+ let root = await readRoot();
+ if (root == null) {
+ const gated = await Delivery().exists({
+ user: input.userId,
+ deliveryKey: input.deliveryKey,
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,
+ });
+ return { status: gated == null ? 'legacy' : 'not_ready' };
+ }
+ if (root.backgroundToolResultBatch?.appliedAt != null) {
+ if (
+ root.backgroundToolResultBatch.proofCopiedAt == null &&
+ !(await confirmAgentBackgroundToolResultBatch({
+ ...input,
+ batchId: root.backgroundToolResultBatch.batchId,
+ }))
+ )
+ return { status: 'not_ready' };
+ return {
+ status: 'claimed',
+ claimId: input.deliveryKey,
+ batchId: root.backgroundToolResultBatch.batchId,
+ ownerStatus: 'applied',
+ };
+ }
+ if (root.backgroundToolResultBatch?.releasing === true) {
+ const released = await releaseAgentBackgroundToolResultClaims({
+ ...input,
+ claimId: input.deliveryKey,
+ batchId: root.backgroundToolResultBatch.batchId,
+ });
+ return released ? claimAgentBackgroundToolResultBatch(input) : { status: 'not_ready' };
+ }
+ if (root.backgroundToolResult == null) return { status: 'not_ready', waitingForResult: true };
+ const claim = root.backgroundToolResult.resultClaim;
+ if (
+ claim?.claimId === input.deliveryKey &&
+ claim.batchId !== root.backgroundToolResultBatch?.batchId
+ ) {
+ await Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
+ 'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,
+ 'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
+ },
+ { $unset: { 'backgroundToolResult.resultClaim': 1 } },
+ { timestamps: false },
+ );
+ return { status: 'not_ready' };
+ }
+ if (claim != null && claim.claimId !== input.deliveryKey) {
+ if (
+ root.backgroundToolResultBatch != null &&
+ root.backgroundToolResultBatch.appliedAt == null
+ ) {
+ await releaseAgentBackgroundToolResultClaims({
+ ...input,
+ claimId: input.deliveryKey,
+ batchId: root.backgroundToolResultBatch.batchId,
+ });
+ }
+ const owner = await Delivery()
+ .findOne({ ...scope, deliveryKey: claim.claimId })
+ .select(projection)
+ .lean<Row>();
+ const batch = owner?.backgroundToolResultBatch;
+ const member = batch?.members?.includes(input.deliveryKey);
+ if (member && batch?.batchId === claim.batchId && batch?.appliedAt != null) {
+ // Retirement removed this owner from the queue. Its followers must
+ // finish the proof copy before they can all settle and abandon it.
+ if (
+ batch.proofCopiedAt == null &&
+ !(await confirmAgentBackgroundToolResultBatch({
+ ...input,
+ deliveryKey: claim.claimId,
+ batchId: batch.batchId,
+ }))
+ )
+ return { status: 'not_ready' };
+ return {
+ status: 'claimed',
+ claimId: claim.claimId,
+ batchId: claim.batchId!,
+ ownerStatus: 'applied',
+ };
+ }
+ if (claim.appliedAt != null)
+ return {
+ status: 'claimed',
+ claimId: claim.claimId,
+ batchId: claim.batchId!,
+ ownerStatus: 'applied',
+ };
+ // A stale collector may finish a CAS after another collector froze a subset.
+ if (
+ (batch?.members != null && !member) ||
+ (batch != null && batch.batchId !== claim.batchId)
+ ) {
+ await Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ 'backgroundToolResult.resultClaim.claimId': claim.claimId,
+ 'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,
+ 'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
+ },
+ { $unset: { 'backgroundToolResult.resultClaim': 1 } },
+ { timestamps: false },
+ );
+ return { status: 'not_ready' };
+ }
+ const retired =
+ owner?.result != null &&
+ typeof owner.result === 'object' &&
+ 'backgroundToolCompletionRetired' in owner.result;
+ const recoverable =
+ owner == null ||
+ DEAD_STATUSES.includes(owner.status) ||
+ owner.capabilityStatus === 'dead' ||
+ retired;
+ return {
+ status: 'claimed',
+ claimId: claim.claimId,
+ batchId: claim.batchId!,
+ ownerStatus: recoverable ? 'recoverable' : 'pending',
+ };
+ }
+ if (root.backgroundToolResultBatch == null) {
+ const candidates =
+ input.limit === 1
+ ? []
+ : await Delivery()
+ .find({
+ ...scope,
+ deliveryKey: { $ne: input.deliveryKey },
+ status: { $in: UNDELIVERED_STATUSES },
+ capabilityStatus: { $ne: 'dead' },
+ backgroundToolResult: { $exists: true },
+ 'backgroundToolResult.resultClaim': { $exists: false },
+ })
+ .select('deliveryKey envelope +backgroundToolResult')
+ .sort({ createdAt: 1, _id: 1 })
+ .limit(Math.min(input.limit, 32) - 1)
+ .lean<Row[]>();
+ const requested = projectBackgroundResult(root);
+ if (requested == null) return { status: 'not_ready' };
+ const metadataCost = (result: NonNullable<ReturnType<typeof projectBackgroundResult>>) =>
+ JSON.stringify(backgroundResultMetadata(result)).length + 1;
+ let remaining = input.maxMetadataChars - metadataCost(requested) - 1;
+ if (remaining < 0) throw new TypeError('Background result metadata exceeds its input budget');
+ const batchId = randomUUID();
+ const keys = [input.deliveryKey];
+ for (const candidate of candidates) {
+ const result = projectBackgroundResult(candidate);
+ if (result == null) continue;
+ const cost = metadataCost(result);
+ if (cost > remaining) continue;
+ remaining -= cost;
+ keys.push(candidate.deliveryKey);
+ }
+ const planned = await Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ backgroundToolResultBatch: { $exists: false },
+ status: { $in: UNDELIVERED_STATUSES },
+ capabilityStatus: { $ne: 'dead' },
+ ...(input.deliveryClaimToken == null
+ ? {}
+ : { capabilityClaimToken: input.deliveryClaimToken }),
+ $or: [
+ { 'backgroundToolResult.resultClaim': { $exists: false } },
+ { 'backgroundToolResult.resultClaim.claimId': input.deliveryKey },
+ ],
+ },
+ { $set: { backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } } },
+ { timestamps: false },
+ );
+ root =
+ planned.modifiedCount === 1
+ ? { ...root, backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } }
+ : await readRoot();
+ if (root == null) return { status: 'not_ready' };
+ }
+ let batch = root.backgroundToolResultBatch;
+ if (batch == null) return { status: 'not_ready' };
+ if (batch.members == null) {
+ // Retirement of a collecting-only plan contends with this root CAS.
+ // No sibling can depend on the owner until its own receipt is claimed.
+ if (root.backgroundToolResult?.resultClaim?.claimId !== input.deliveryKey) {
+ const claimedRoot = await Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ status: { $in: UNDELIVERED_STATUSES },
+ capabilityStatus: { $ne: 'dead' },
+ 'backgroundToolResultBatch.batchId': batch.batchId,
+ 'backgroundToolResultBatch.releasing': { $ne: true },
+ 'backgroundToolResult.resultClaim': { $exists: false },
+ ...(input.deliveryClaimToken == null
+ ? {}
+ : { capabilityClaimToken: input.deliveryClaimToken }),
+ },
+ {
+ $set: {
+ 'backgroundToolResult.resultClaim': {
+ kind: 'wakeup',
+ claimId: input.deliveryKey,
+ claimedAt: new Date(),
+ batchId: batch.batchId,
+ },
+ },
+ },
+ { timestamps: false },
+ );
+ if (claimedRoot.matchedCount !== 1) return { status: 'not_ready' };
+ }
+ await Delivery().updateMany(
+ {
+ ...scope,
+ deliveryKey: { $in: batch.candidates },
+ status: { $in: UNDELIVERED_STATUSES },
+ capabilityStatus: { $ne: 'dead' },
+ backgroundToolResult: { $exists: true },
+ 'backgroundToolResult.resultClaim': { $exists: false },
+ },
+ {
+ $set: {
+ 'backgroundToolResult.resultClaim': {
+ kind: 'wakeup',
+ claimId: input.deliveryKey,
+ claimedAt: new Date(),
+ batchId: batch.batchId,
+ },
+ },
+ },
+ { timestamps: false },
+ );
+ const owned = await Delivery()
+ .find({
+ ...scope,
+ deliveryKey: { $in: batch.candidates },
+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
+ 'backgroundToolResult.resultClaim.batchId': batch.batchId,
+ })
+ .select('deliveryKey')
+ .lean<Pick<IAgentTriggerDelivery, 'deliveryKey'>[]>();
+ const owners = new Set(owned.map((row) => row.deliveryKey));
+ const members = batch.candidates.filter((key) => owners.has(key));
+ if (!owners.has(input.deliveryKey)) {
+ await releaseAgentBackgroundToolResultClaims({
+ ...input,
+ claimId: input.deliveryKey,
+ batchId: batch.batchId,
+ });
+ return { status: 'not_ready' };
+ }
+ const frozen = await Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
+ 'backgroundToolResultBatch.batchId': batch.batchId,
+ 'backgroundToolResultBatch.releasing': { $ne: true },
+ ...(input.deliveryClaimToken == null
+ ? {}
+ : { capabilityClaimToken: input.deliveryClaimToken }),
+ 'backgroundToolResultBatch.members': { $exists: false },
+ },
+ { $set: { 'backgroundToolResultBatch.members': members } },
+ { timestamps: false },
+ );
+ if (frozen.modifiedCount === 1) {
+ batch = { ...batch, members };
+ } else {
+ root = await readRoot();
+ batch = root?.backgroundToolResultBatch;
+ }
+ }
+ if (batch?.members == null || !batch.members.includes(input.deliveryKey))
+ return { status: 'not_ready' };
+ const rows = await Delivery()
+ .find({
+ ...scope,
+ deliveryKey: { $in: batch.members },
+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
+ 'backgroundToolResult.resultClaim.batchId': batch.batchId,
+ })
+ .select('deliveryKey envelope +backgroundToolResult')
+ .lean<Row[]>();
+ const byKey = new Map(rows.map((row) => [row.deliveryKey, projectBackgroundResult(row)]));
+ const results = batch.members.map((key) => byKey.get(key));
+ if (results.some((result) => result == null)) return { status: 'not_ready' };
+ if (
+ input.deliveryClaimToken != null &&
(await Delivery().exists({
...scope,
- 'backgroundToolResult.resultClaim.claimId': input.claimId,
+ deliveryKey: input.deliveryKey,
+ capabilityClaimToken: input.deliveryClaimToken,
+ 'backgroundToolResultBatch.batchId': batch.batchId,
+ 'backgroundToolResultBatch.releasing': { $ne: true },
})) == null
+ ) {
+ return { status: 'not_ready' };
+ }
+ return {
+ status: 'acquired',
+ batchId: batch.batchId,
+ results: results.filter((result): result is NonNullable<typeof result> => result != null),
+ };
+ }
+
+ async function getAgentBackgroundToolResultBatch(
+ input: AgentBackgroundToolResultBatchOwnerInput,
+ ): Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null> {
+ const row = await Delivery()
+ .findOne({ ...backgroundResultIdentity(input), deliveryKey: input.deliveryKey })
+ .select('+backgroundToolResultBatch')
+ .lean<Pick<IAgentTriggerDelivery, 'backgroundToolResultBatch'>>();
+ return row?.backgroundToolResultBatch ?? null;
+ }
+
+ async function beginAgentBackgroundToolResultBatchDispatch(
+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },
+ ): Promise<boolean> {
+ const marked = await Delivery().updateOne(
+ {
+ ...backgroundResultIdentity(input),
+ deliveryKey: input.deliveryKey,
+ status: { $in: UNDELIVERED_STATUSES },
+ capabilityStatus: { $ne: 'dead' },
+ 'backgroundToolResultBatch.batchId': input.batchId,
+ 'backgroundToolResultBatch.members.0': { $exists: true },
+ 'backgroundToolResultBatch.releasing': { $ne: true },
+ 'backgroundToolResultBatch.appliedAt': { $exists: false },
+ ...(input.deliveryClaimToken == null
+ ? {}
+ : { capabilityClaimToken: input.deliveryClaimToken }),
+ },
+ {
+ $inc: { 'backgroundToolResultBatch.dispatchCount': 1 },
+ $set: { 'backgroundToolResultBatch.dispatchId': input.dispatchId },
+ },
+ { timestamps: false },
+ );
+ return marked.matchedCount === 1;
+ }
+
+ /** Admission proof precedes follower settlement, including lost HTTP replies. */
+ async function confirmAgentBackgroundToolResultBatch(
+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },
+ ): Promise<boolean> {
+ const scope = backgroundResultIdentity(input);
+ const root = await Delivery()
+ .findOneAndUpdate(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ 'backgroundToolResultBatch.members.0': { $exists: true },
+ 'backgroundToolResultBatch.batchId': input.batchId,
+ 'backgroundToolResultBatch.releasing': { $ne: true },
+ },
+ { $min: { 'backgroundToolResultBatch.appliedAt': new Date() } },
+ { new: true, timestamps: false },
+ )
+ .select('status +backgroundToolResultBatch')
+ .lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();
+ if (root?.backgroundToolResultBatch?.appliedAt == null) return false;
+ await Delivery().updateMany(
+ {
+ ...scope,
+ deliveryKey: { $in: root.backgroundToolResultBatch.members },
+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
+ 'backgroundToolResult.resultClaim.batchId': input.batchId,
+ },
+ {
+ $set: {
+ 'backgroundToolResult.resultClaim.appliedAt': root.backgroundToolResultBatch.appliedAt,
+ },
+ },
+ { timestamps: false },
);
+ if (
+ (await Delivery().exists({
+ ...scope,
+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,
+ 'backgroundToolResult.resultClaim.batchId': input.batchId,
+ 'backgroundToolResult.resultClaim.appliedAt': { $exists: false },
+ })) != null
+ )
+ return false;
+ const finish = (retired: boolean) =>
+ Delivery().updateOne(
+ {
+ ...scope,
+ deliveryKey: input.deliveryKey,
+ 'backgroundToolResultBatch.batchId': input.batchId,
+ 'backgroundToolResultBatch.appliedAt': { $exists: true },
+ status: retired ? 'succeeded' : { $ne: 'succeeded' },
+ },
+ {
+ $min: { 'backgroundToolResultBatch.proofCopiedAt': new Date() },
+ ...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),
+ },
+ { timestamps: false },
+ );
+ const finished = await finish(root.status === 'succeeded');
+ if (finished.matchedCount === 1) return true;
+ return root.status !== 'succeeded' && (await finish(true)).matchedCount === 1;
}
async function settleAgentTriggerHandlingOutcome(
@@ -3715,8 +4361,7 @@ export function createAgentTriggerDeliveryMethods(
const error = normalizeFailure(input.error);
const receiptFence = recoverReceipt
? {
- requiredWorkerCapability:
- AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
backgroundToolResult: { $exists: true },
}
: {};
@@ -4273,7 +4918,11 @@ export function createAgentTriggerDeliveryMethods(
},
{
$set: { backgroundToolResultErasedAt: erasedAt },
- $unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },
+ $unset: {
+ backgroundToolResult: 1,
+ backgroundToolResultBatch: 1,
+ backgroundToolResultDeletionPendingAt: 1,
+ },
},
{ timestamps: false },
);
@@ -4287,7 +4936,7 @@ export function createAgentTriggerDeliveryMethods(
await Delivery().updateMany(
{
user,
- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },
'envelope.target.conversationId': { $in: conversationIds },
backgroundToolResultErasedAt: { $exists: false },
},
@@ -4316,6 +4965,10 @@ export function createAgentTriggerDeliveryMethods(
getAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
claimAgentBackgroundToolResults,
+ claimAgentBackgroundToolResultBatch,
+ confirmAgentBackgroundToolResultBatch,
+ beginAgentBackgroundToolResultBatchDispatch,
+ getAgentBackgroundToolResultBatch,
releaseAgentBackgroundToolResultClaims,
settleAgentTriggerHandlingOutcome,
admitAgentEventActorAction,
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
index 927e421b3d..4db14e4d96 100644
--- a/packages/data-schemas/src/schema/triggerDelivery.ts
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -168,6 +168,8 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
kind: { type: String, enum: ['wakeup'], required: true },
claimId: { type: String, required: true, maxlength: 128 },
claimedAt: { type: Date, required: true },
+ appliedAt: { type: Date },
+ batchId: { type: String, maxlength: 128 },
},
{ _id: false },
),
@@ -178,6 +180,23 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
),
select: false,
},
+ backgroundToolResultBatch: {
+ type: new Schema(
+ {
+ batchId: { type: String, required: true, maxlength: 128 },
+ dispatchCount: { type: Number, required: true, min: 0 },
+ dispatchId: { type: String, maxlength: 128 },
+ releaseId: { type: String, maxlength: 128 },
+ proofCopiedAt: { type: Date },
+ candidates: { type: [String], required: true },
+ releasing: { type: Boolean },
+ members: { type: [String], default: undefined },
+ appliedAt: { type: Date },
+ },
+ { _id: false },
+ ),
+ select: false,
+ },
backgroundToolResultErasedAt: { type: Date, select: false },
backgroundToolResultDeletionPendingAt: { type: Date, select: false },
attempts: { type: Number, required: true, default: 0, min: 0 },
diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts
index 73ede79450..694dfc4991 100644
--- a/packages/data-schemas/src/types/triggerDelivery.ts
+++ b/packages/data-schemas/src/types/triggerDelivery.ts
@@ -17,6 +17,8 @@ export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
'background_tool_completion_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 =
'background_tool_completion_receipt_v2';
+export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 =
+ 'background_tool_completion_batch_v3';
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2 = 'agent_queued_turn_v2';
export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;
@@ -92,6 +94,8 @@ export interface AgentBackgroundToolResultReceipt {
kind: 'wakeup';
claimId: string;
claimedAt: Date;
+ appliedAt?: Date;
+ batchId?: string;
};
}
@@ -128,6 +132,18 @@ export interface IAgentTriggerDelivery {
producerLeaseUntil?: Date;
/** Durable source of truth for a background completion. */
backgroundToolResult?: AgentBackgroundToolResultReceipt;
+ /** Receipt batching is separate from event-envelope coalescing. */
+ backgroundToolResultBatch?: {
+ batchId: string;
+ dispatchCount: number;
+ dispatchId?: string;
+ releaseId?: string;
+ proofCopiedAt?: Date;
+ candidates: string[];
+ members?: string[];
+ appliedAt?: Date;
+ releasing?: true;
+ };
/** Conversation-deletion fence preventing a late producer from restoring private output. */
backgroundToolResultErasedAt?: Date;
backgroundToolResultDeletionPendingAt?: Date;