Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🪇 perf: Coalesce Ready Background Completions (#16650)
lia-by-librechat[bot]f88f3fb2febb
diff --git a/api/server/experimental.js b/api/server/experimental.jsindex 18983dafb8..2ad6e559fa 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -709,6 +709,8 @@ if (cluster.isMaster) {address: server.address(),completionResultBatchSize:baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,+ completionReceiptBatching:+ baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling,});} catch (initErr) {diff --git a/api/server/index.js b/api/server/index.jsindex eec43ca22e..4ceba2caff 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -508,6 +508,8 @@ const startServer = async () => {address: server.address(),completionResultBatchSize:appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,+ completionReceiptBatching:+ appConfig?.endpoints?.agents?.backgroundTasks?.completionReceiptBatching,idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling,});const scheduleEngineArmed = (await initializeScheduleEngine()) != null;diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.jsindex f8944c6ea8..3977b8e3f8 100644--- a/api/server/services/Agents/triggers.js+++ b/api/server/services/Agents/triggers.js@@ -6,6 +6,7 @@ const {createSubagentCompletionWakeupResolver,SUBAGENT_COMPLETION_SOURCE,createBackgroundToolCompletionWakeupResolver,+ createBackgroundToolDeadClaimRecovery,BACKGROUND_TOOL_COMPLETION_SOURCE,createAgentQueuedTurnLifecycle,AGENT_QUEUED_TURN_SOURCE,@@ -30,6 +31,22 @@ const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupResomethods,getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),+ getGenerationAdmissionEvidence,+ recoverDeadClaim: createBackgroundToolDeadClaimRecovery(+ (...args) => service.retire(...args),+ methods.releaseBackgroundToolResultClaims,+ (conversationId) => GenerationJobManager.getJob(conversationId),+ ({ userId, conversationId, claimId }) =>+ GenerationJobManager.fenceGenerationClaimForRecovery(+ userId,+ claimId,+ conversationId,+ conversationId,+ ),+ methods.releaseAgentBackgroundToolResultClaims,+ methods,+ getGenerationAdmissionEvidence,+ ),getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),});const eventActorAdapter = createAgentEventContinueResolver({@@ -95,6 +112,7 @@ module.exports = {persistAgentBackgroundToolResult: service.persistBackgroundToolResult,expediteCompletionWakeups: service.expediteCompletionWakeups,getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,+ getBackgroundCompletionReceiptBatching: service.getBackgroundCompletionReceiptBatching,releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,drainAgentTriggerDeliveriesForUser: service.drainUser,prepareAgentTriggerUserPurge: service.prepareUserPurge,diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.jsindex df3a09b2d6..ed99ccab28 100644--- a/api/server/services/Agents/triggers.spec.js+++ b/api/server/services/Agents/triggers.spec.js@@ -19,6 +19,7 @@ const mockQueuedTurnLifecycle = {jest.mock('@librechat/api', () => ({createCheckpointDeletionReclaimer: jest.fn((getJobs) => () => getJobs('owner', 'tenant')),+ createBackgroundToolDeadClaimRecovery: jest.fn(() => jest.fn()),createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),createAgentContinuationResolver: jest.fn(() => jest.fn()),createAgentEventContinueResolver: jest.fn(() => jest.fn()),diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.jsindex 94d1f52eec..88ea7c4fa8 100644--- a/api/server/services/Endpoints/agents/backgroundCompletion.js+++ b/api/server/services/Endpoints/agents/backgroundCompletion.js@@ -1,5 +1,6 @@const {createBackgroundToolCompletionWakeupHandler,+ GenerationJobManager,createBackgroundToolDeadClaimRecovery,createPendingBackgroundCompletions,createBackgroundToolResultHandler,@@ -7,12 +8,15 @@ const {} = require('@librechat/api');const {listPendingAgentBackgroundToolCompletions,+ getAgentBackgroundToolResultBatch,+ confirmAgentBackgroundToolResultBatch,listUndeliveredAgentTriggerTaskIds,} = require('~/models');const {enqueueAgentTrigger,persistAgentBackgroundToolResult,getAgentBackgroundToolResultClaim,+ getBackgroundCompletionReceiptBatching,releaseAgentBackgroundToolResultClaims,renewAgentTriggerProducerLease,retireAgentTrigger,@@ -26,6 +30,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup(deliveryKey, sourceId, result) =>persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),(deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }),+ getBackgroundCompletionReceiptBatching,);const pendingBackgroundToolCompletions = createPendingBackgroundCompletions({@@ -52,6 +57,8 @@ function createDeadBackgroundToolClaimRecovery(getGenerationJob,fenceGenerationClaim,releaseAgentBackgroundToolResultClaims,+ { getAgentBackgroundToolResultBatch, confirmAgentBackgroundToolResultBatch },+ (...args) => GenerationJobManager.getGenerationAdmissionEvidence(...args),);}diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.tsindex 571dca62f6..ead7e37788 100644--- a/packages/api/src/agents/background.spec.ts+++ b/packages/api/src/agents/background.spec.ts@@ -53,6 +53,42 @@ async function waitForSubagentTaskToSettle(throw new Error('Timed out waiting for the detached subagent task.');}+describe('manual reconciliation restoration', () => {+ it('upgrades only the confirmed local owner and preserves the marker on replay', () => {+ const registry = new BackgroundTaskRegistryClass();+ const created = registry.create({+ userId: 'marker-user',+ conversationId: 'marker-convo',+ toolCallId: 'marker-call',+ toolName: 'tool',+ });+ if ('atCapacity' in created) throw new Error('Unexpected capacity');+ registry.complete('marker-user', 'marker-convo', created.task.id, { content: 'done' });+ const claim = { kind: 'manual' as const, claimId: 'poll' };+ expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(+ 'acquired',+ );+ expect(+ registry.claimResult('marker-user', 'marker-convo', created.task.id, {+ ...claim,+ claimId: 'foreign',+ receiptReconciled: true,+ }),+ ).toBe('claimed');+ expect(created.task.resultClaim?.receiptReconciled).toBeUndefined();+ expect(+ registry.claimResult('marker-user', 'marker-convo', created.task.id, {+ ...claim,+ receiptReconciled: true,+ }),+ ).toBe('replay');+ expect(registry.claimResult('marker-user', 'marker-convo', created.task.id, claim)).toBe(+ 'replay',+ );+ expect(created.task.resultClaim?.receiptReconciled).toBe(true);+ });+});+describe('isBackgroundEligibleToolName', () => {it('excludes direct-path, host-special, and machinery tools', () => {for (const name of [diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.tsindex f98f0b3406..43cfb15893 100644--- a/packages/api/src/agents/background.ts+++ b/packages/api/src/agents/background.ts@@ -668,6 +668,7 @@ export interface BackgroundTask {claimId: string;claimedAt: number;generationId?: string;+ receiptReconciled?: true;};/** The declared tool may return a process-local live artifact. A terminal* same-generation poll may therefore deliver from the local claim after it@@ -1823,7 +1824,12 @@ export class BackgroundTaskRegistryClass {userId: string,conversationId: string,taskId: string,- claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string },+ claim: {+ kind: 'manual' | 'wakeup';+ claimId: string;+ generationId?: string;+ receiptReconciled?: true;+ },): 'acquired' | 'replay' | 'claimed' | 'not_ready' {const task = this.get(userId, conversationId, taskId);if (task == null || task.status === 'running') {@@ -1834,9 +1840,14 @@ export class BackgroundTaskRegistryClass {task.updatedAt = Date.now();return 'acquired';}- return task.resultClaim.kind === claim.kind && task.resultClaim.claimId === claim.claimId- ? 'replay'- : 'claimed';+ if (task.resultClaim.kind !== claim.kind || task.resultClaim.claimId !== claim.claimId) {+ return 'claimed';+ }+ if (claim.receiptReconciled === true && task.resultClaim.receiptReconciled !== true) {+ task.resultClaim.receiptReconciled = true;+ task.updatedAt = Date.now();+ }+ return 'replay';}releaseResultClaim(@@ -2554,6 +2565,7 @@ export async function runCheckBackgroundTask(params: {conversationId,messageId: task.messageId,claimId: existingClaim.claimId,+ ...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),...(existingClaim.kind === 'manual'? {kind: 'manual' as const,@@ -2674,6 +2686,7 @@ export async function runCheckBackgroundTask(params: {taskId,{kind: 'manual',+ receiptReconciled: true,claimId: invocationId,...(params.generationId == null ? {} : { generationId: params.generationId }),},@@ -2744,6 +2757,7 @@ export async function runCheckBackgroundTask(params: {taskId,{kind: 'manual',+ receiptReconciled: true,claimId: invocationId,...(params.generationId == null ? {} : { generationId: params.generationId }),},@@ -2854,6 +2868,7 @@ export async function runCheckBackgroundTask(params: {conversationId,messageId: claimedResult.messageId,claimId: existingClaim.claimId,+ ...(existingClaim.batchId != null && { batchId: existingClaim.batchId }),...(existingClaim.kind === 'manual'? {kind: 'manual' as const,@@ -3242,6 +3257,7 @@ export function getBackgroundCodeDelivery(params: {kind: task.resultClaim.kind,claimId: task.resultClaim.claimId,claimedAt: new Date(task.resultClaim.claimedAt),+ ...(task.resultClaim.receiptReconciled === true && { receiptReconciled: true }),...(task.resultClaim.generationId == null? {}: { generationId: task.resultClaim.generationId }),diff --git a/packages/api/src/agents/backgroundClaims.ts b/packages/api/src/agents/backgroundClaims.tsindex 6f92bca763..320aab3994 100644--- a/packages/api/src/agents/backgroundClaims.ts+++ b/packages/api/src/agents/backgroundClaims.ts@@ -4,7 +4,11 @@ import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup'/** Reconciles manual polling with the independent automatic-delivery receipt,* including polls reconstructed after the process-local registry was lost. */export async function claimBackgroundToolResult(- methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,+ methods: Pick<+ MessageMethods,+ 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'+ > &+ Partial<Pick<MessageMethods, 'confirmBackgroundToolResultClaim'>>,getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],): ReturnType<MessageMethods['claimBackgroundToolResults']> {@@ -30,7 +34,7 @@ export async function claimBackgroundToolResult(if (messageId == null) {throw new Error('The background result claim has no parent message identity');}- const release = async () => {+ const release = async (allowReconciled = false) => {const released = await methods.releaseBackgroundToolResultClaims({userId: input.userId,conversationId: input.conversationId,@@ -38,8 +42,10 @@ export async function claimBackgroundToolResult(taskIds: messageClaim.results.map((result) => result.taskId),kind: input.kind,claimId: input.claimId,+ ...(input.batchId != null && { batchId: input.batchId }),+ ...(input.kind === 'manual' && { onlyIfUnreconciled: true }),});- if (!released) {+ if (!released && !allowReconciled) {throw new Error('The background result claim could not be released');}};@@ -51,6 +57,26 @@ export async function claimBackgroundToolResult(throw error;}if (receiptClaim == null || receiptClaim.claimId === input.claimId) {+ if (input.kind === 'manual' && methods.confirmBackgroundToolResultClaim != null) {+ try {+ if (+ !(await methods.confirmBackgroundToolResultClaim({+ userId: input.userId,+ conversationId: input.conversationId,+ messageId,+ taskId: input.taskId,+ claimId: input.claimId,+ }))+ ) {+ throw new Error('The reconciled background result claim was superseded');+ }+ } catch (error) {+ // This CAS removes only speculative ownership. A committed or unknown+ // handoff remains owned for generation-fenced recovery.+ await release(true);+ throw error;+ }+ }return messageClaim;}await release();diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.tsindex 8886ea89c0..e04f82a262 100644--- a/packages/api/src/agents/backgroundCompletion.ts+++ b/packages/api/src/agents/backgroundCompletion.ts@@ -65,6 +65,9 @@ export interface BackgroundToolDeadClaimRecoveryInput {conversationId: string;messageId: string;claimId: string;+ batchId?: string;+ /** Automatic repair cannot reopen a committed manual handoff. */+ onlyIfUnreconciled?: true;/** Omitted for the legacy automatic-wakeup recovery path. */kind?: 'manual' | 'wakeup';/** Required to prove that a manual claim's owning generation is no longer active. */diff --git a/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.tsnew file mode 100644index 0000000000..5b06bdcb73--- /dev/null+++ b/packages/api/src/agents/backgroundCompletionWakeup.integration.spec.ts@@ -0,0 +1,1479 @@+import mongoose from 'mongoose';+import { MongoMemoryServer } from 'mongodb-memory-server';+import {+ createMethods,+ createModels,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+} from '@librechat/data-schemas';+import type { IMessage } from '@librechat/data-schemas';+import type { AgentTriggerFetch } from './triggers/host';+import {+ createBackgroundToolCompletionWakeupResolver,+ createBackgroundToolDeadClaimRecovery,+ BACKGROUND_TOOL_COMPLETION_SOURCE,+} from './backgroundCompletionWakeup';+import {+ backgroundTaskRegistry,+ runCheckBackgroundTask,+ getBackgroundCodeDelivery,+} from './background';+import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './triggers/envelope';+import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';+import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';+import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';+import { createAgentTriggerExecutionHost } from './triggers/host';+import { prepareAgentTriggerDelivery } from './triggers/delivery';+import { claimBackgroundToolResult } from './backgroundClaims';++let mongo: MongoMemoryServer;+let methods: ReturnType<typeof createMethods>;+let userId: string;+const conversationId = 'receipt-replay';+const parentMessageId = 'receipt-parent';+const agentId = 'agent_parent_1';+const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3;++beforeAll(async () => {+ mongo = await MongoMemoryServer.create();+ await mongoose.connect(mongo.getUri());+ createModels(mongoose);+ methods = createMethods(mongoose);+}, 60_000);++afterAll(async () => {+ await mongoose.disconnect();+ await mongo.stop();+});++beforeEach(async () => {+ await Promise.all(+ ['AgentTriggerDelivery', 'AgentTriggerLaneSequence', 'Conversation', 'Message'].map((name) =>+ mongoose.models[name].deleteMany({}),+ ),+ );+ userId = new mongoose.Types.ObjectId().toString();+ await mongoose.models.Conversation.create({+ conversationId,+ user: userId,+ tenantId: 'tenant-1',+ endpoint: 'agents',+ agent_id: agentId,+ });+ await mongoose.models.Message.create({+ messageId: parentMessageId,+ conversationId,+ user: userId,+ tenantId: 'tenant-1',+ parentMessageId: 'user-parent',+ isCreatedByUser: false,+ unfinished: false,+ endpoint: 'agents',+ content: [],+ });+});++async function ready(taskId: string, persistReceipt = true) {+ const envelope = createAgentTriggerEnvelope({+ mode: 'continue',+ requestId: taskId,+ deliveryId: taskId,+ receivedAt: Date.now(),+ principal: { id: userId, tenantId: 'tenant-1' },+ event: {+ id: taskId,+ type: 'background-tool.completion',+ occurredAt: Date.now(),+ source: { id: BACKGROUND_TOOL_COMPLETION_SOURCE, type: 'internal' },+ payload: { taskId, toolCallId: taskId, toolName: 'tool' },+ },+ target: { conversationId, parentMessageId, agentId },+ input: 'waiting',+ });+ const row = prepareAgentTriggerDelivery(envelope, {+ orderingKey: taskId,+ requiredWorkerCapability: capability,+ });+ await methods.enqueueAgentTriggerDelivery(row);+ if (persistReceipt)+ await methods.persistAgentBackgroundToolResult({+ deliveryKey: row.deliveryKey,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ result: { status: 'completed', output: taskId, settledAt: new Date() },+ });+ return envelope;+}++function owner(deliveryKey: string) {+ return {+ deliveryKey,+ userId,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ tenantId: 'tenant-1',+ conversationId,+ parentMessageId,+ agentId,+ };+}++it('replays exactly the admitted input after a lost response and a definitely rejected retry', async () => {+ const root = await ready('one');+ const sibling = await ready('two');+ const inputs: string[] = [];+ const consumed: string[] = [];+ let firstInput: string | undefined;+ let calls = 0;+ const fetcher: AgentTriggerFetch = async (_url, init) => {+ const body = JSON.parse(String(init?.body)) as { text: string };+ inputs.push(body.text);+ if (++calls === 1) {+ firstInput = body.text;+ consumed.push('one', 'two');+ throw Object.assign(new Error('lost admitted response'), { code: 'ECONNRESET' });+ }+ if (calls === 2)+ throw Object.assign(new Error('retry did not connect'), { code: 'ECONNREFUSED' });+ if (body.text !== firstInput) consumed.push('three');+ return new Response(+ JSON.stringify({+ status: 'started',+ streamId: conversationId,+ conversationId,+ generationCreatedAt: 100,+ }),+ { status: 200 },+ );+ };+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ const host = createAgentTriggerExecutionHost({+ prepareContinue: resolve,+ fetch: fetcher,+ mintToken: () => 'test',+ getBaseUrl: () => 'http://localhost',+ });+ const options = { requiredWorkerCapability: capability };+ await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'ambiguous' });+ const late = await ready('three');+ await expect(host.dispatch(root, options)).rejects.toMatchObject({ certainty: 'definite' });+ await expect(host.dispatch(root, options)).resolves.toMatchObject({ status: 'started' });+ expect(inputs).toEqual([firstInput, firstInput, firstInput]);+ expect(consumed).toEqual(['one', 'two']);+ await expect(host.dispatch(sibling, options)).resolves.toMatchObject({ status: 'settled' });+ const lateEnvelope = late.mode === 'continue' ? late : undefined;+ if (lateEnvelope == null) throw new Error('Expected continuation');+ const prepared = await resolve(lateEnvelope, {+ idempotencyKey: getAgentTriggerIdempotencyKey(late),+ ...options,+ });+ expect(prepared?.status === 'ready' && prepared.input).toContain('three');+});++it('cleans a late predecessor projection without clearing its successor receipt batch', async () => {+ const root = await ready('one');+ const deliveryKey = getAgentTriggerIdempotencyKey(root);+ const scope = owner(deliveryKey);+ const old = await methods.claimAgentBackgroundToolResultBatch({+ ...scope,+ limit: 8,+ maxMetadataChars: 16000,+ });+ if (old.status !== 'acquired') throw new Error('Expected batch');+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...scope,+ claimId: deliveryKey,+ batchId: old.batchId,+ }),+ ).toBe(true);+ await mongoose.models.Message.updateOne(+ { messageId: parentMessageId, user: userId },+ {+ $set: {+ content: [+ {+ type: 'tool_call',+ tool_call: {+ id: 'one',+ output: 'one',+ backgroundTask: {+ taskId: 'one',+ toolName: 'tool',+ status: 'completed',+ completionWakeup: true,+ completionReceipt: true,+ },+ },+ },+ ],+ },+ },+ );+ // A predecessor's delayed projection CAS lands after its cleanup completed.+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'wakeup',+ claimId: deliveryKey,+ batchId: old.batchId,+ limit: 1,+ });+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ await expect(+ resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }),+ ).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING' });+ const successor = await methods.getAgentBackgroundToolResultBatch(scope);+ expect(successor?.batchId).not.toBe(old.batchId);+ await expect(+ resolve(root, { idempotencyKey: deliveryKey, requiredWorkerCapability: capability }),+ ).resolves.toMatchObject({ status: 'ready' });+ expect((await methods.getAgentBackgroundToolResultBatch(scope))?.batchId).toBe(+ successor?.batchId,+ );+});++it('confirms a dead owner with native admission proof instead of re-presenting its results', async () => {+ const root = await ready('one');+ const sibling = await ready('two');+ const deliveryKey = getAgentTriggerIdempotencyKey(root);+ const scope = owner(deliveryKey);+ await methods.claimAgentBackgroundToolResultBatch({+ ...scope,+ limit: 8,+ maxMetadataChars: 16000,+ });+ await mongoose.models.AgentTriggerDelivery.updateOne(+ { deliveryKey },+ { $set: { capabilityStatus: 'dead' } },+ );+ const recover = createBackgroundToolDeadClaimRecovery(+ async (key, sourceId, reason, options) =>+ methods.retireAgentTriggerDelivery({+ deliveryKey: key,+ sourceId,+ reason,+ settledAt: new Date(),+ ...options,+ }),+ methods.releaseBackgroundToolResultClaims,+ async () => null,+ async () => 'started',+ methods.releaseAgentBackgroundToolResultClaims,+ methods,+ async () => ({ generationId: conversationId, generationCreatedAt: 100 }),+ );+ expect(+ await recover({ userId, conversationId, messageId: parentMessageId, claimId: deliveryKey }),+ ).toBe(false);+ const follower = await methods.claimAgentBackgroundToolResultBatch({+ ...owner(getAgentTriggerIdempotencyKey(sibling)),+ limit: 8,+ maxMetadataChars: 16000,+ });+ expect(follower).toMatchObject({ status: 'claimed', ownerStatus: 'applied' });+});++async function project(taskId: string) {+ await mongoose.models.Message.updateOne(+ { messageId: parentMessageId, user: userId },+ {+ $push: {+ content: {+ type: 'tool_call',+ tool_call: {+ id: taskId,+ output: `durable-${taskId}`,+ backgroundTask: {+ taskId,+ toolName: 'tool',+ status: 'completed',+ settledAt: new Date(),+ completionWakeup: true,+ completionReceipt: true,+ },+ },+ },+ },+ },+ );+}++it('reconstructs a failed receipt write from the successful terminal message projection', async () => {+ const root = await ready('one', false);+ const deliveryKey = getAgentTriggerIdempotencyKey(root);+ await project('one');+ await mongoose.models.AgentTriggerDelivery.updateOne(+ { deliveryKey },+ { $set: { producerLeaseUntil: new Date(0) } },+ );+ const fetcher: AgentTriggerFetch = async (_url, init) => {+ expect(JSON.parse(String(init?.body)).text).toContain('durable-one');+ return new Response(+ JSON.stringify({+ status: 'started',+ conversationId,+ streamId: conversationId,+ generationCreatedAt: 100,+ }),+ { status: 200 },+ );+ };+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ const host = createAgentTriggerExecutionHost({+ prepareContinue: resolve,+ fetch: fetcher,+ mintToken: () => 'test',+ getBaseUrl: () => 'http://localhost',+ });+ await expect(+ host.dispatch(root, { requiredWorkerCapability: capability }),+ ).resolves.toMatchObject({ status: 'started' });+ expect(+ await methods.getAgentBackgroundToolResult({+ deliveryKey,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ }),+ ).toMatchObject({ output: 'durable-one', resultClaim: { appliedAt: expect.any(Date) } });+});++it('does not settle a root when a speculative manual poll and automatic owner mutually yield', async () => {+ const root = await ready('one');+ await ready('two');+ await project('one');+ const key = getAgentTriggerIdempotencyKey(root);+ let entered: () => void = () => undefined;+ let captured: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const automaticEntered = new Promise<void>((resolve) => {+ entered = resolve;+ });+ const snapshotTaken = new Promise<void>((resolve) => {+ captured = resolve;+ });+ const manualMayYield = new Promise<void>((resolve) => {+ resume = resolve;+ });+ let lookups = 0;+ const getter = async (input: Parameters<typeof methods.getAgentBackgroundToolResultClaim>[0]) => {+ if (++lookups === 1) return methods.getAgentBackgroundToolResultClaim(input);+ await automaticEntered;+ const snapshot = await methods.getAgentBackgroundToolResultClaim(input);+ captured();+ await manualMayYield;+ return snapshot;+ };+ const claim = methods.claimBackgroundToolResults.bind(methods);+ const automatic = jest+ .spyOn(methods, 'claimBackgroundToolResults')+ .mockImplementation(async (input) => {+ if (input.kind === 'wakeup' && input.taskId === 'one') {+ entered();+ await snapshotTaken;+ }+ return claim(input);+ });+ const manual = claimBackgroundToolResult(methods, getter, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ generationId: 'manual-generation',+ });+ // Allow the initial manual projection CAS to commit before receipt selection.+ const deadline = Date.now() + 3000;+ while (+ (await mongoose.models.Message.findOne({+ messageId: parentMessageId,+ 'content.tool_call.backgroundTask.resultClaim.claimId': 'manual-poll',+ }).lean()) == null+ ) {+ if (Date.now() > deadline) throw new Error('Manual claim did not arrive');+ await new Promise((resolve) => setTimeout(resolve, 5));+ }+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ await expect(+ resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability }),+ ).rejects.toMatchObject({ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING', deferWithoutAttempt: true });+ resume();+ expect(await manual).toMatchObject({ status: 'claimed' });+ automatic.mockRestore();+ const retry = await resolve(root, { idempotencyKey: key, requiredWorkerCapability: capability });+ expect(retry?.status === 'ready' && retry.input).toContain('one');+ expect(retry?.status === 'ready' && retry.input).toContain('two');+});++it('settles a root only after a manual poll has completed receipt reconciliation', async () => {+ const root = await ready('one');+ await project('one');+ expect(+ (+ await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ })+ ).status,+ ).toBe('acquired');+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ await expect(+ resolve(root, {+ idempotencyKey: getAgentTriggerIdempotencyKey(root),+ requiredWorkerCapability: capability,+ }),+ ).resolves.toEqual({ status: 'settled' });+});++it.each([false, true])(+ 'rolls back an unconfirmed manual claim with generation identity %s',+ async (withGeneration) => {+ const root = await ready('one');+ await project('one');+ const confirmation = jest+ .spyOn(mongoose.models.Message, 'updateOne')+ .mockImplementationOnce(() => {+ throw new Error('confirmation unavailable');+ });+ await expect(+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ ...(withGeneration && { generationId: 'manual-generation' }),+ }),+ ).rejects.toThrow('confirmation unavailable');+ confirmation.mockRestore();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ const prepared = await resolve(root, {+ idempotencyKey: getAgentTriggerIdempotencyKey(root),+ requiredWorkerCapability: capability,+ });+ expect(prepared?.status === 'ready' && prepared.input).toContain('"result":"one"');+ },+);++it('resolves a committed manual confirmation whose write reply was lost', async () => {+ const root = await ready('one');+ await project('one');+ const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message);+ const lostReply = jest+ .spyOn(mongoose.models.Message, 'updateOne')+ .mockImplementationOnce((...args) => {+ const query = update(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ await execute();+ throw new Error('lost confirmation reply');+ });+ return query;+ });+ const consumed = await claimBackgroundToolResult(+ methods,+ methods.getAgentBackgroundToolResultClaim,+ {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ generationId: 'manual-generation',+ },+ );+ expect(consumed).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });+ lostReply.mockRestore();+ expect(+ await methods.releaseBackgroundToolResultClaims({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskIds: ['one'],+ kind: 'manual',+ claimId: 'manual-poll',+ onlyIfUnreconciled: true,+ }),+ ).toBe(false);+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ await expect(+ resolve(root, {+ idempotencyKey: getAgentTriggerIdempotencyKey(root),+ requiredWorkerCapability: capability,+ }),+ ).resolves.toEqual({ status: 'settled' });+});++it('recovers abandoned manual ownership after both confirmation and rollback failed', async () => {+ const root = await ready('one');+ await ready('two');+ await project('one');+ const confirmation = jest+ .spyOn(methods, 'confirmBackgroundToolResultClaim')+ .mockRejectedValueOnce(new Error('confirmation unavailable'));+ const rollback = jest+ .spyOn(methods, 'releaseBackgroundToolResultClaims')+ .mockRejectedValueOnce(new Error('rollback unavailable'));+ await expect(+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ generationId: 'manual-generation',+ }),+ ).rejects.toThrow('rollback unavailable');+ confirmation.mockRestore();+ rollback.mockRestore();+ let active = true;+ const getGenerationJob = async () =>+ active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null;+ const recover = createBackgroundToolDeadClaimRecovery(+ async () => false,+ methods.releaseBackgroundToolResultClaims,+ getGenerationJob,+ async () => 'unavailable',+ );+ const recoverDeadClaim = jest.fn(recover);+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob,+ recoverDeadClaim,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ const key = getAgentTriggerIdempotencyKey(root);+ await mongoose.models.AgentTriggerDelivery.updateOne(+ { deliveryKey: key },+ { $set: { capabilityClaimToken: 'exact-queue-lease' } },+ );+ const context = {+ idempotencyKey: key,+ requiredWorkerCapability: capability,+ deliveryClaimToken: 'exact-queue-lease',+ };+ await expect(resolve(root, context)).rejects.toMatchObject({ code: 'PARENT_NOT_READY' });+ expect(recoverDeadClaim).not.toHaveBeenCalled();+ active = false;+ await expect(resolve(root, context)).rejects.toMatchObject({+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ deferWithoutAttempt: true,+ });+ expect(recoverDeadClaim).toHaveBeenCalledWith(+ expect.objectContaining({+ kind: 'manual',+ claimId: 'manual-poll',+ generationId: 'manual-generation',+ }),+ );+ const prepared = await resolve(root, context);+ expect(prepared?.status === 'ready' && prepared.input).toContain('one');+ expect(prepared?.status === 'ready' && prepared.input).toContain('two');+});++it('does not use a different manual owner as confirmation read-back proof', async () => {+ await ready('one');+ await project('one');+ const update = mongoose.models.Message.updateOne.bind(mongoose.models.Message);+ const replaced = jest+ .spyOn(mongoose.models.Message, 'updateOne')+ .mockImplementationOnce((...args) => {+ const query = update(...args);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ await update(+ { user: userId, messageId: parentMessageId },+ {+ $set: {+ 'content.0.tool_call.backgroundTask.resultClaim': {+ kind: 'manual',+ claimId: 'successor-poll',+ receiptReconciled: true,+ },+ },+ },+ );+ throw new Error('predecessor confirmation failed');+ });+ return query;+ });+ await expect(+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ }),+ ).rejects.toThrow('predecessor confirmation failed');+ replaced.mockRestore();+ expect(+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'wakeup',+ claimId: 'automatic',+ }),+ ).toMatchObject({+ status: 'claimed',+ claim: { claimId: 'successor-poll', receiptReconciled: true },+ });+});++it.each(['missing-receipt', 'batch-root', 'batch-sibling'] as const)(+ 'preserves a manual handoff committed after the %s recovery snapshot',+ async (scenario) => {+ const polled = await ready('one', scenario !== 'missing-receipt');+ const other = scenario === 'batch-sibling' ? await ready('two') : undefined;+ const root = other ?? polled;+ await project('one');+ let confirmationReached: () => void = () => undefined;+ let confirmNow: () => void = () => undefined;+ let recoveryReached: () => void = () => undefined;+ let recoverNow: () => void = () => undefined;+ const confirmationEntered = new Promise<void>((resolve) => {+ confirmationReached = resolve;+ });+ const confirmationBarrier = new Promise<void>((resolve) => {+ confirmNow = resolve;+ });+ const recoveryEntered = new Promise<void>((resolve) => {+ recoveryReached = resolve;+ });+ const recoveryBarrier = new Promise<void>((resolve) => {+ recoverNow = resolve;+ });+ const confirm = methods.confirmBackgroundToolResultClaim.bind(methods);+ const pausedConfirmation = jest+ .spyOn(methods, 'confirmBackgroundToolResultClaim')+ .mockImplementationOnce(async (input) => {+ confirmationReached();+ await confirmationBarrier;+ return confirm(input);+ });+ const manual = claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ generationId: 'manual-generation',+ });+ await confirmationEntered;+ let manualActive = true;+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);+ const recover = createBackgroundToolDeadClaimRecovery(+ async () => false,+ release,+ async () =>+ manualActive+ ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } }+ : null,+ async () => 'unavailable',+ );+ const recoverDeadClaim = jest.fn(async (input: Parameters<typeof recover>[0]) => {+ recoveryReached();+ await recoveryBarrier;+ return recover(input);+ });+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ recoverDeadClaim,+ });+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ const key = getAgentTriggerIdempotencyKey(root);+ await mongoose.models.AgentTriggerDelivery.updateMany(+ { deliveryKey: { $in: [key, getAgentTriggerIdempotencyKey(polled)] } },+ {+ $set: {+ capabilityStatus: 'leased',+ capabilityClaimToken: 'exact-queue-lease',+ capabilityLeaseBy: 'automatic-worker',+ capabilityLeaseUntil: new Date(Date.now() + 60_000),+ },+ },+ );+ const context = {+ idempotencyKey: key,+ requiredWorkerCapability: capability,+ deliveryClaimToken: 'exact-queue-lease',+ };+ const automatic = resolve(root, context);+ await recoveryEntered;+ confirmNow();+ expect(await manual).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });+ // The automatic queue lease won manual retirement. The poll returned its+ // result and its generation ended while recovery held a stale snapshot.+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: getAgentTriggerIdempotencyKey(polled),+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ settledAt: new Date(),+ reason: 'manual poll',+ onlyIfUnclaimed: true,+ requireTransition: true,+ }),+ ).toBe(false);+ manualActive = false;+ recoverNow();+ await expect(automatic).rejects.toMatchObject({+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ deferWithoutAttempt: true,+ });+ expect(recoverDeadClaim).toHaveBeenCalledWith(+ expect.objectContaining({ onlyIfUnreconciled: true }),+ );+ expect(release).toHaveBeenCalledWith(+ expect.objectContaining({ kind: 'manual', claimId: 'manual-poll', onlyIfUnreconciled: true }),+ );+ const claim = await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'wakeup',+ claimId: 'probe',+ });+ expect(claim).toMatchObject({+ status: 'claimed',+ claim: { claimId: 'manual-poll', receiptReconciled: true },+ });+ const pollEnvelope = polled.mode === 'continue' ? polled : undefined;+ if (pollEnvelope == null) throw new Error('Expected polled continuation');+ await expect(+ resolve(pollEnvelope, {+ idempotencyKey: getAgentTriggerIdempotencyKey(polled),+ requiredWorkerCapability: capability,+ }),+ ).resolves.toEqual({ status: 'settled' });+ if (other != null) {+ const row = await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(polled));+ if (row == null) throw new Error('Expected polled delivery');+ expect(+ await methods.completeAgentTriggerDelivery({+ id: row.id,+ workerId: 'automatic-worker',+ claimToken: 'exact-queue-lease',+ attempt: 1,+ settledAt: new Date(),+ result: { status: 'settled', mode: 'continue', conversationId },+ }),+ ).toBe(true);+ const remaining = await resolve(root, context);+ if (remaining?.status !== 'ready') throw new Error('Expected undelivered sibling');+ expect(remaining.input).toContain('"background_task_id":"two"');+ expect(remaining.input).not.toContain('"background_task_id":"one"');+ await remaining.beginDispatch?.();+ }+ pausedConfirmation.mockRestore();+ },+);++it('keeps explicit manual recovery able to reopen a committed claim after its generation ends', async () => {+ await ready('one');+ await project('one');+ await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'manual-poll',+ generationId: 'manual-generation',+ });+ let active = true;+ const recover = createBackgroundToolDeadClaimRecovery(+ async () => false,+ methods.releaseBackgroundToolResultClaims,+ async () =>+ active ? { status: 'running', metadata: { responseMessageId: 'manual-generation' } } : null,+ async () => 'unavailable',+ );+ const input = {+ userId,+ conversationId,+ messageId: parentMessageId,+ claimId: 'manual-poll',+ kind: 'manual' as const,+ generationId: 'manual-generation',+ };+ expect(await recover(input)).toBe(false);+ active = false;+ expect(await recover({ ...input, onlyIfUnreconciled: true })).toBe(false);+ expect(await recover(input)).toBe(true);+ expect(+ (+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'manual',+ claimId: 'next-poll',+ })+ ).status,+ ).toBe('acquired');+});++it.each(['same-generation', 'later-generation'] as const)(+ 'restores manual reconciliation after %s final persistence',+ async (scenario) => {+ const generationId = scenario === 'same-generation' ? parentMessageId : 'manual-generation';+ const created = backgroundTaskRegistry.create({+ userId,+ conversationId,+ toolCallId: 'manual-final-call',+ toolName: 'tool',+ messageId: parentMessageId,+ harvestStarted: true,+ });+ if ('atCapacity' in created) throw new Error('Unexpected task capacity');+ const taskId = created.task.id;+ const root = await ready(taskId);+ await project(taskId);+ await mongoose.models.Message.updateOne(+ { user: userId, messageId: parentMessageId },+ { $set: { unfinished: true } },+ );+ const original = await mongoose.models.Message.findOne({+ user: userId,+ messageId: parentMessageId,+ }).lean<Pick<IMessage, 'content'>>();+ backgroundTaskRegistry.complete(userId, conversationId, taskId, { content: 'manual-result' });+ backgroundTaskRegistry.finishHarvest(userId, conversationId, taskId);+ backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, taskId, {+ renew: async () => true,+ retire: async (reason, options) =>+ methods.retireAgentTriggerDelivery({+ deliveryKey: getAgentTriggerIdempotencyKey(root),+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ settledAt: new Date(),+ reason,+ ...options,+ }),+ });+ const polled = JSON.parse(+ await runCheckBackgroundTask({+ userId,+ conversationId,+ args: { background_task_id: taskId },+ toolCallId: 'confirmed-poll',+ runId: 'manual-run',+ generationId,+ claimBackgroundToolResult: (input) =>+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),+ }),+ );+ expect(polled).toMatchObject({ status: 'completed', result: 'manual-result' });+ expect(backgroundTaskRegistry.get(userId, conversationId, taskId)?.resultClaim).toMatchObject({+ receiptReconciled: true,+ });+ // Final full persistence uses content captured before manual confirmation.+ await mongoose.models.Message.updateOne(+ { user: userId, messageId: parentMessageId },+ { $set: { unfinished: false, content: original?.content } },+ );+ const restored = getBackgroundCodeDelivery({+ userId,+ conversationId,+ args: { background_task_id: taskId },+ });+ expect(restored?.backgroundTask?.resultClaim).toMatchObject({ receiptReconciled: true });+ await methods.updateToolCallResult({+ userId,+ conversationId,+ messageId: parentMessageId,+ toolCallId: taskId,+ output: 'manual-result',+ backgroundTask: restored?.backgroundTask,+ });+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);+ const recover = createBackgroundToolDeadClaimRecovery(+ async () => false,+ release,+ async () => null,+ async () => 'unavailable',+ );+ const claim = await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId,+ kind: 'wakeup',+ claimId: 'automatic-probe',+ });+ if (claim.status !== 'claimed' || claim.claim == null)+ throw new Error('Expected preserved manual claim');+ expect(claim.claim).toMatchObject({ receiptReconciled: true });+ expect(+ await recover({+ userId,+ conversationId,+ messageId: parentMessageId,+ claimId: claim.claim.claimId,+ kind: 'manual',+ generationId,+ onlyIfUnreconciled: true,+ }),+ ).toBe(false);+ expect(+ (await methods.getAgentTriggerDelivery(getAgentTriggerIdempotencyKey(root)))?.status,+ ).toBe('succeeded');+ },+);++it.each(['admitted', 'unpublished'] as const)(+ 'uses native recovery proof after custom-store %s job cleanup',+ async (state) => {+ const root = await ready('one');+ const sibling = await ready('two');+ const key = getAgentTriggerIdempotencyKey(root);+ await methods.claimAgentBackgroundToolResultBatch({+ ...owner(key),+ limit: 8,+ maxMetadataChars: 16000,+ });+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });+ // Historical lookup is optional; the native claim/fence CAS still exists.+ Reflect.set(store, 'getIdempotencyClaim', undefined);+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore: store,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ try {+ const claimed = await manager.claimGeneration(userId, key, conversationId, conversationId, 2);+ if (state === 'admitted') {+ const job = await manager.createJob(conversationId, userId, conversationId, {+ idempotencyClientRequestId: key,+ idempotencyClaimToken: claimed.existing!.claimToken,+ initialMetadata: { generationProtocolVersion: 2 },+ });+ expect(await store.deleteJob(conversationId, job.createdAt)).toBe(true);+ }+ expect(+ await manager.getGenerationAdmissionEvidence(userId, key, conversationId, conversationId),+ ).toBeNull();+ await mongoose.models.AgentTriggerDelivery.updateOne(+ { deliveryKey: key },+ { $set: { capabilityStatus: 'dead' } },+ );+ const release = jest.fn(methods.releaseAgentBackgroundToolResultClaims);+ const recover = createBackgroundToolDeadClaimRecovery(+ async (deliveryKey, sourceId, reason, options) =>+ methods.retireAgentTriggerDelivery({+ deliveryKey,+ sourceId,+ reason,+ settledAt: new Date(),+ ...options,+ }),+ methods.releaseBackgroundToolResultClaims,+ async () => null,+ ({ userId, conversationId, claimId }) =>+ manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId),+ release,+ methods,+ (...args) => manager.getGenerationAdmissionEvidence(...args),+ );+ expect(+ await recover({ userId, conversationId, messageId: parentMessageId, claimId: key }),+ ).toBe(state === 'unpublished');+ if (state === 'admitted') {+ expect(release).not.toHaveBeenCalled();+ expect(+ await methods.claimAgentBackgroundToolResultBatch({+ ...owner(getAgentTriggerIdempotencyKey(sibling)),+ limit: 8,+ maxMetadataChars: 16000,+ }),+ ).toMatchObject({ status: 'claimed', ownerStatus: 'applied' });+ const batch = await methods.getAgentBackgroundToolResultBatch(owner(key));+ expect(batch?.proofCopiedAt).toBeInstanceOf(Date);+ } else {+ expect(release).toHaveBeenCalledWith(expect.objectContaining({ recoveryFenced: true }));+ expect(+ (+ await methods.claimAgentBackgroundToolResultBatch({+ ...owner(getAgentTriggerIdempotencyKey(sibling)),+ limit: 8,+ maxMetadataChars: 16000,+ })+ ).status,+ ).toBe('acquired');+ }+ } finally {+ await manager.destroy();+ }+ },+);++it.each(['lookup', 'confirmation', 'confirmation-unknown'] as const)(+ 'preserves a committed manual replay when %s is unavailable',+ async (phase) => {+ const root = await ready('one');+ await project('one');+ const key = getAgentTriggerIdempotencyKey(root);+ await mongoose.models.AgentTriggerDelivery.updateOne(+ { deliveryKey: key },+ { $set: { capabilityStatus: 'leased', capabilityClaimToken: 'automatic-lease' } },+ );+ const input = {+ userId,+ conversationId,+ taskId: 'one',+ kind: 'manual' as const,+ claimId: 'committed-poll',+ generationId: 'manual-generation',+ };+ const delivered = await claimBackgroundToolResult(+ methods,+ methods.getAgentBackgroundToolResultClaim,+ input,+ );+ expect(delivered).toMatchObject({ status: 'acquired' });+ let restore: () => void = () => undefined;+ try {+ if (phase === 'lookup') {+ await expect(+ claimBackgroundToolResult(+ methods,+ async () => {+ throw new Error('receipt lookup unavailable');+ },+ input,+ ),+ ).rejects.toThrow();+ } else if (phase === 'confirmation') {+ const failure = jest+ .spyOn(mongoose.models.Message, 'updateOne')+ .mockImplementationOnce(() => {+ throw new Error('confirmation write unavailable');+ });+ restore = () => {+ failure.mockRestore();+ };+ // Exact committed read-back succeeds even though this replay's write fails.+ await expect(+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ ...input,+ generationId: undefined,+ }),+ ).resolves.toEqual(delivered);+ } else {+ const failure = jest+ .spyOn(methods, 'confirmBackgroundToolResultClaim')+ .mockRejectedValueOnce(new Error('confirmation outcome unavailable'));+ restore = () => {+ failure.mockRestore();+ };+ await expect(+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),+ ).rejects.toThrow('confirmation outcome unavailable');+ }+ } finally {+ restore();+ }+ const competitor = await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'one',+ kind: 'wakeup',+ claimId: 'automatic-probe',+ });+ expect(competitor).toMatchObject({+ status: 'claimed',+ claim: { claimId: input.claimId, generationId: 'manual-generation', receiptReconciled: true },+ });+ expect(+ await methods.releaseBackgroundToolResultClaims({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskIds: ['one'],+ kind: 'manual',+ claimId: input.claimId,+ onlyIfUnreconciled: true,+ }),+ ).toBe(false);+ if (root.mode !== 'continue') throw new Error('Expected continuation');+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ await expect(+ resolve(root, {+ idempotencyKey: key,+ requiredWorkerCapability: capability,+ deliveryClaimToken: 'automatic-lease',+ }),+ ).resolves.toEqual({ status: 'settled' });+ },+);++it('repairs native started-fence confirmation from an applied follower after a retired owner crashed', async () => {+ const root = await ready('one');+ const sibling = await ready('two');+ const key = getAgentTriggerIdempotencyKey(root);+ const claimed = await methods.claimAgentBackgroundToolResultBatch({+ ...owner(key),+ limit: 8,+ maxMetadataChars: 16000,+ });+ if (claimed.status !== 'acquired') throw new Error('Expected frozen batch');+ await mongoose.models.AgentTriggerDelivery.updateOne(+ { deliveryKey: key },+ { $set: { capabilityStatus: 'dead' } },+ );+ const recover = createBackgroundToolDeadClaimRecovery(+ async (deliveryKey, sourceId, reason, options) =>+ methods.retireAgentTriggerDelivery({+ deliveryKey,+ sourceId,+ reason,+ settledAt: new Date(),+ ...options,+ }),+ methods.releaseBackgroundToolResultClaims,+ async () => null,+ async () => 'started',+ methods.releaseAgentBackgroundToolResultClaims,+ methods,+ async () => null,+ );+ const Delivery = mongoose.models.AgentTriggerDelivery;+ const copyFailure = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {+ throw new Error('proof copy lost');+ });+ await expect(+ recover({ userId, conversationId, messageId: parentMessageId, claimId: key }),+ ).rejects.toThrow('proof copy lost');+ copyFailure.mockRestore();+ expect(await Delivery.findOne({ deliveryKey: key }).lean()).toMatchObject({+ status: 'succeeded',+ });+ expect(await Delivery.findOne({ deliveryKey: key }).lean()).not.toHaveProperty('expiresAt');+ if (sibling.mode !== 'continue') throw new Error('Expected sibling');+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods,+ getGenerationJob: async () => null,+ });+ await expect(+ resolve(sibling, {+ idempotencyKey: getAgentTriggerIdempotencyKey(sibling),+ requiredWorkerCapability: capability,+ }),+ ).resolves.toEqual({ status: 'settled' });+ expect(+ (await methods.getAgentBackgroundToolResultBatch(owner(key)))?.proofCopiedAt,+ ).toBeInstanceOf(Date);+ expect(await Delivery.findOne({ deliveryKey: key }).lean()).toHaveProperty('expiresAt');+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'one' }),+ ).toMatchObject({ appliedAt: expect.any(Date) });+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...owner(key), taskId: 'two' }),+ ).toMatchObject({ appliedAt: expect.any(Date) });+});++it.each([+ ['local', false],+ ['reconstructed', false],+ ['local', true],+ ['reconstructed', true],+] as const)(+ 'recovers a released physical projection through a %s poll after successor retirement %s',+ async (pollPath, replaceOwner) => {+ const root = await ready('stale-task');+ const key = getAgentTriggerIdempotencyKey(root);+ const scope = owner(key);+ await project('stale-task');+ const old = await methods.claimAgentBackgroundToolResultBatch({+ ...scope,+ limit: 8,+ maxMetadataChars: 16000,+ });+ if (old.status !== 'acquired') throw new Error('Expected predecessor batch');+ let pause: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const paused = new Promise<void>((resolve) => {+ pause = resolve;+ });+ const barrier = new Promise<void>((resolve) => {+ resume = resolve;+ });+ const Message = mongoose.models.Message;+ const update = Message.findOneAndUpdate.bind(Message);+ const lateProjection = jest+ .spyOn(Message, 'findOneAndUpdate')+ .mockImplementationOnce((...args) => {+ const query = update(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ pause();+ await barrier;+ return execute();+ });+ return query;+ });+ const delayed = methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'stale-task',+ kind: 'wakeup',+ claimId: key,+ batchId: old.batchId,+ limit: 1,+ });+ await paused;+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...scope,+ claimId: key,+ batchId: old.batchId,+ }),+ ).toBe(true);+ if (replaceOwner) {+ const successor = await methods.claimAgentBackgroundToolResultBatch({+ ...scope,+ limit: 8,+ maxMetadataChars: 16000,+ });+ if (successor.status !== 'acquired') throw new Error('Expected successor batch');+ expect(successor.batchId).not.toBe(old.batchId);+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...scope,+ claimId: key,+ batchId: successor.batchId,+ }),+ ).toBe(true);+ }+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: key,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ reason: 'unadmitted owner retired',+ settledAt: new Date(),+ onlyIfUnclaimed: true,+ }),+ ).toBe(true);+ resume();+ await expect(delayed).resolves.toMatchObject({ status: 'acquired' });+ lateProjection.mockRestore();+ expect(await methods.getAgentBackgroundToolResultBatch(scope)).toBeNull();++ // A broad logical-owner release would also erase these unrelated claims.+ await project('other-epoch');+ await project('manual-task');+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'other-epoch',+ kind: 'wakeup',+ claimId: key,+ batchId: 'unrelated-epoch',+ limit: 1,+ });+ await claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, {+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'manual-task',+ kind: 'manual',+ claimId: key,+ });+ if (pollPath === 'local') {+ const created = backgroundTaskRegistry.create({+ taskId: 'stale-task',+ userId,+ conversationId,+ messageId: parentMessageId,+ toolCallId: 'stale-task',+ toolName: 'tool',+ });+ if ('atCapacity' in created) throw new Error('Unexpected capacity');+ backgroundTaskRegistry.complete(userId, conversationId, created.task.id, {+ content: 'durable-stale-task',+ });+ backgroundTaskRegistry.markCompletionWakeup(userId, conversationId, created.task.id);+ }+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore: store,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ try {+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);+ const recover = createBackgroundToolDeadClaimRecovery(+ async (deliveryKey, sourceId, reason, options) =>+ methods.retireAgentTriggerDelivery({+ deliveryKey,+ sourceId,+ reason,+ settledAt: new Date(),+ ...options,+ }),+ release,+ async () => null,+ ({ userId, conversationId, claimId }) =>+ manager.fenceGenerationClaimForRecovery(userId, claimId, conversationId, conversationId),+ methods.releaseAgentBackgroundToolResultClaims,+ methods,+ (...args) => manager.getGenerationAdmissionEvidence(...args),+ );+ const recoverDeadBackgroundToolClaim = jest.fn(recover);+ const polled = JSON.parse(+ await runCheckBackgroundTask({+ userId,+ conversationId,+ args: { background_task_id: 'stale-task' },+ toolCallId: 'stale-poll',+ runId: 'recovery-run',+ generationId: 'recovery-generation',+ claimBackgroundToolResult: (input) =>+ claimBackgroundToolResult(methods, methods.getAgentBackgroundToolResultClaim, input),+ recoverDeadBackgroundToolClaim,+ }),+ );+ expect(polled).toMatchObject({ status: 'completed', result: 'durable-stale-task' });+ expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith(+ expect.objectContaining({ claimId: key, batchId: old.batchId }),+ );+ expect(release).toHaveBeenCalledWith(+ expect.objectContaining({ kind: 'wakeup', claimId: key, batchId: old.batchId }),+ );+ for (const [taskId, claim] of [+ ['other-epoch', { kind: 'wakeup', claimId: key, batchId: 'unrelated-epoch' }],+ ['manual-task', { kind: 'manual', claimId: key, receiptReconciled: true }],+ ] as const) {+ expect(+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId,+ kind: 'wakeup',+ claimId: 'probe',+ }),+ ).toMatchObject({ status: 'claimed', claim });+ }+ } finally {+ await manager.destroy();+ }+ },+);++it('does not release a stale epoch while a successor receipt batch owns the logical delivery', async () => {+ const root = await ready('stale-task');+ const key = getAgentTriggerIdempotencyKey(root);+ const scope = owner(key);+ await project('stale-task');+ const old = await methods.claimAgentBackgroundToolResultBatch({+ ...scope,+ limit: 8,+ maxMetadataChars: 16000,+ });+ if (old.status !== 'acquired') throw new Error('Expected predecessor');+ await methods.releaseAgentBackgroundToolResultClaims({+ ...scope,+ claimId: key,+ batchId: old.batchId,+ });+ const successor = await methods.claimAgentBackgroundToolResultBatch({+ ...scope,+ limit: 8,+ maxMetadataChars: 16000,+ });+ if (successor.status !== 'acquired') throw new Error('Expected successor');+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'stale-task',+ kind: 'wakeup',+ claimId: key,+ batchId: successor.batchId,+ limit: 1,+ });+ const release = jest.fn(methods.releaseBackgroundToolResultClaims);+ const retire = jest.fn(async () => true);+ const fence = jest.fn(async () => 'fenced' as const);+ const recover = createBackgroundToolDeadClaimRecovery(+ retire,+ release,+ async () => null,+ fence,+ methods.releaseAgentBackgroundToolResultClaims,+ methods,+ );+ expect(+ await recover({+ userId,+ conversationId,+ messageId: parentMessageId,+ claimId: key,+ batchId: old.batchId,+ }),+ ).toBe(false);+ expect(retire).not.toHaveBeenCalled();+ expect(fence).not.toHaveBeenCalled();+ expect(release).not.toHaveBeenCalled();+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...scope, taskId: 'stale-task' }),+ ).toMatchObject({ batchId: successor.batchId });+ expect(+ await methods.claimBackgroundToolResults({+ userId,+ conversationId,+ messageId: parentMessageId,+ taskId: 'stale-task',+ kind: 'manual',+ claimId: 'probe',+ }),+ ).toMatchObject({ status: 'claimed', claim: { claimId: key, batchId: successor.batchId } });+});diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.tsindex 3d8c9460a1..f599dad7a6 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts@@ -1,4 +1,7 @@-import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';+import {+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+} from '@librechat/data-schemas';import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';import type { CodeApprovalMode } from 'librechat-data-provider';import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';@@ -113,6 +116,37 @@ describe('background tool completion wakeups', () => {jest.useRealTimers();});+ it('admits v3 receipts only when the deployment rollout gate is enabled', async () => {+ const enqueue = jest.fn<+ ReturnType<EnqueueBackgroundToolCompletion>,+ Parameters<EnqueueBackgroundToolCompletion>+ >(async () => ({ deliveryKey: 'key' }));+ let enabled = false;+ const notify = createBackgroundToolCompletionWakeupHandler(+ enqueue,+ async () => true,+ async () => true,+ undefined,+ undefined,+ () => enabled,+ );+ await notify(registration());+ expect(enqueue).toHaveBeenLastCalledWith(+ expect.anything(),+ expect.objectContaining({+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ }),+ );+ enabled = true;+ await notify(registration({ taskId: 'task-2' }));+ expect(enqueue).toHaveBeenLastCalledWith(+ expect.anything(),+ expect.objectContaining({+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+ }),+ );+ });+it('expedites its own delivery when a result it can consume appears', async () => {const expedite = jest.fn();const notify = createBackgroundToolCompletionWakeupHandler(@@ -265,6 +299,7 @@ describe('background tool completion wakeups', () => {userId: 'user-1',conversationId: 'conversation-1',messageId: 'response-1',+ allowMissingMessage: true,kind: 'wakeup',claimId: 'batch-root-delivery',});@@ -274,6 +309,7 @@ describe('background tool completion wakeups', () => {conversationId: 'conversation-1',parentMessageId: 'response-1',claimId: 'batch-root-delivery',+ recoveryFenced: true,});});@@ -1034,3 +1070,204 @@ describe('pending background completions', () => {}});});++describe('capability-gated receipt batch resolution', () => {+ function batchMethods() {+ const { methods } = resolverMethods();+ return {+ ...methods,+ claimAgentBackgroundToolResultBatch: jest.fn(async () => ({+ status: 'acquired' as const,+ batchId: 'batch-1',+ results: [+ {+ taskId: 'task-1',+ toolCallId: 'call-1',+ toolName: 'tool',+ status: 'completed' as const,+ output: 'one',+ },+ {+ taskId: 'task-2',+ toolCallId: 'call-2',+ toolName: 'tool',+ status: 'completed' as const,+ output: 'two',+ },+ ],+ })),+ beginAgentBackgroundToolResultBatchDispatch: jest.fn(async () => true),+ confirmAgentBackgroundToolResultBatch: jest.fn(async () => true),+ };+ }++ it('does not probe batch storage for rollout-default v2 deliveries', async () => {+ const methods = batchMethods();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ await resolve(await envelope(), {+ idempotencyKey: 'delivery-1',+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ });+ expect(methods.claimAgentBackgroundToolResultBatch).not.toHaveBeenCalled();+ });++ it('reconciles every member before dispatch and confirms only after admission', async () => {+ const methods = batchMethods();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });+ expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({ taskId: 'task-1' }),+ );+ expect(methods.claimBackgroundToolResults).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ taskId: 'task-2' }),+ );+ expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();+ expect(methods.claimAgentBackgroundToolResultBatch).toHaveBeenCalledWith(+ expect.objectContaining({+ tenantId: 'tenant-1',+ limit: 8,+ maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,+ }),+ );+ if (prepared?.status !== 'ready') throw new Error('Expected ready');+ expect(prepared.input).toContain('task-2');+ await prepared.settleOnAdmission?.({+ mode: 'continue',+ status: 'started',+ conversationId: 'conversation-1',+ });+ expect(methods.confirmAgentBackgroundToolResultBatch).toHaveBeenCalledTimes(1);+ });++ it('defers followers until the root has confirmed admission', async () => {+ const methods = batchMethods();+ methods.claimAgentBackgroundToolResultBatch+ .mockResolvedValueOnce({+ status: 'claimed',+ claimId: 'root',+ ownerStatus: 'pending',+ } as never)+ .mockResolvedValueOnce({+ status: 'claimed',+ claimId: 'root',+ ownerStatus: 'applied',+ } as never);+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({+ code: 'BACKGROUND_TOOL_BATCH_PENDING',+ deferWithoutAttempt: true,+ });+ await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).resolves.toEqual({+ status: 'settled',+ });+ expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();+ });++ it('uses fenced dead-owner recovery before retrying a follower as its own root', async () => {+ const methods = batchMethods();+ const recoverDeadClaim = jest.fn(async () => true);+ methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({+ status: 'claimed',+ claimId: 'root',+ ownerStatus: 'recoverable',+ } as never);+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ recoverDeadClaim,+ });+ await expect(resolve(await envelope(), { idempotencyKey: 'follower' })).rejects.toMatchObject({+ code: 'BACKGROUND_TOOL_BATCH_PENDING',+ });+ expect(recoverDeadClaim).toHaveBeenCalledWith(+ expect.objectContaining({ claimId: 'root', messageId: 'response-1' }),+ );+ expect((await resolve(await envelope(), { idempotencyKey: 'follower' }))?.status).toBe('ready');+ });++ it('releases the entire frozen batch when a sibling manual poll wins', async () => {+ const methods = batchMethods();+ methods.claimBackgroundToolResults+ .mockResolvedValueOnce({ status: 'acquired', results: [] })+ .mockResolvedValueOnce({+ status: 'claimed',+ claim: { kind: 'manual', claimId: 'poll' },+ } as never);+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ });+ expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(+ expect.objectContaining({ claimId: 'root' }),+ );+ expect(methods.confirmAgentBackgroundToolResultBatch).not.toHaveBeenCalled();+ });++ it('releases receipt ownership even when no message projection has landed', async () => {+ const methods = batchMethods();+ methods.claimBackgroundToolResults.mockResolvedValue({ status: 'not_found' } as never);+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });+ if (prepared?.status !== 'ready') throw new Error('Expected ready');+ await prepared.releaseOnDefiniteFailure?.();+ expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();+ expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalled();+ });++ it('retains ownership when admission confirmation fails after dispatch', async () => {+ const methods = batchMethods();+ methods.confirmAgentBackgroundToolResultBatch.mockRejectedValueOnce(new Error('lost reply'));+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ const prepared = await resolve(await envelope(), { idempotencyKey: 'root' });+ if (prepared?.status !== 'ready') throw new Error('Expected ready');+ await expect(+ prepared.settleOnAdmission?.({+ mode: 'continue',+ status: 'started',+ conversationId: 'conversation-1',+ }),+ ).rejects.toThrow('lost reply');+ expect(methods.releaseAgentBackgroundToolResultClaims).not.toHaveBeenCalled();+ });++ it('still detects a lost producer while a v3 receipt is not ready', async () => {+ const methods = batchMethods();+ methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });+ methods.claimAgentBackgroundToolResultBatch.mockResolvedValueOnce({+ status: 'not_ready',+ waitingForResult: true,+ } as never);+ methods.getAgentTriggerDeliveryProducerLease.mockResolvedValueOnce({+ status: 'expired',+ leaseUntil: new Date(0),+ });+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ await expect(resolve(await envelope(), { idempotencyKey: 'root' })).rejects.toMatchObject({+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',+ retryable: false,+ });+ });+});diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.tsindex e3df97eb04..56a057f95d 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts@@ -1,6 +1,10 @@import { randomUUID } from 'node:crypto';import { backgroundResultMetadata, isEphemeralAgentId } from 'librechat-data-provider';-import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';+import {+ AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+} from '@librechat/data-schemas';import type {AgentTriggerProducerLeaseStatus,AgentTriggerDeliveryMethods,@@ -80,6 +84,10 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &output: string;settledAt: Date;} | null>;+ persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];+ beginAgentBackgroundToolResultBatchDispatch?: AgentTriggerDeliveryMethods['beginAgentBackgroundToolResultBatchDispatch'];+ claimAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResultBatch'];+ confirmAgentBackgroundToolResultBatch?: AgentTriggerDeliveryMethods['confirmAgentBackgroundToolResultBatch'];claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];@@ -98,6 +106,13 @@ export interface BackgroundToolCompletionWakeupResolverDeps {methods: WakeupMethods;getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;getResultBatchSize?: () => number | undefined;+ recoverDeadClaim?: BackgroundToolDeadClaimRecovery;+ getGenerationAdmissionEvidence?: (+ userId: string,+ clientRequestId: string,+ streamId: string,+ conversationId?: string,+ ) => Promise<{ generationId: string; generationCreatedAt: number } | null>;/** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */getWaitMaxIntervalMs?: () => number | undefined;}@@ -265,12 +280,14 @@ export function createBackgroundToolCompletionWakeupResolver({getGenerationJob,getResultBatchSize,getWaitMaxIntervalMs,+ recoverDeadClaim,+ getGenerationAdmissionEvidence,}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<AgentTriggerExecutionHostDeps['prepareContinue']> {const waitingRetry = (receivedAt: number): string =>waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);- return async (+ const resolve = async (envelope: AgentContinueTriggerEnvelope,context: AgentTriggerDispatchContext,): Promise<AgentTriggerContinuePreparation | undefined> => {@@ -332,6 +349,313 @@ export function createBackgroundToolCompletionWakeupResolver({status: 404,});}+ const batchScope = {+ deliveryKey: context.idempotencyKey,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ userId,+ ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),+ conversationId: envelope.target.conversationId,+ parentMessageId: envelope.target.parentMessageId,+ agentId: envelope.target.agentId,+ ...(context.deliveryClaimToken != null && { deliveryClaimToken: context.deliveryClaimToken }),+ };+ const batch =+ context.requiredWorkerCapability != null &&+ context.requiredWorkerCapability !==+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3+ ? undefined+ : await methods.claimAgentBackgroundToolResultBatch?.({+ ...batchScope,+ limit: getResultBatchSize?.() ?? 8,+ maxMetadataChars: BACKGROUND_TOOL_WAKEUP_INPUT_MAX_CHARS - 256,+ });+ if (batch?.status === 'claimed') {+ if (batch.ownerStatus === 'applied') return { status: 'settled' };+ if (batch.ownerStatus === 'recoverable') {+ await recoverDeadClaim?.({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ claimId: batch.claimId,+ batchId: batch.batchId,+ });+ }+ throw executionError('The background result is owned by another completion delivery.', {+ code: 'BACKGROUND_TOOL_BATCH_PENDING',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: '1',+ });+ }+ if (batch?.status === 'not_ready') {+ if (batch.waitingForResult === true) {+ // A producer can fail its receipt write while its terminal message+ // projection succeeds. Reconstruct from that already-durable result.+ const projection = await methods.claimBackgroundToolResults({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ taskId: registration.taskId,+ agentId: envelope.target.agentId,+ kind: 'wakeup',+ claimId: context.idempotencyKey,+ limit: 1,+ });+ if (+ projection.status === 'claimed' &&+ projection.claim?.kind === 'manual' &&+ projection.claim.receiptReconciled === true+ )+ return { status: 'settled' };+ if (projection.status === 'acquired') {+ const result = projection.results.find((result) => result.taskId === registration.taskId);+ const releaseProjection = async (): Promise<void> => {+ if (+ !(await methods.releaseBackgroundToolResultClaims({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ taskIds: [registration.taskId],+ kind: 'wakeup',+ claimId: context.idempotencyKey,+ allowMissingMessage: true,+ }))+ ) {+ throw new Error('Background projection reconstruction release was not confirmed');+ }+ };+ let restored = false;+ try {+ if (result != null)+ restored =+ (await methods.persistAgentBackgroundToolResult?.({+ deliveryKey: context.idempotencyKey,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ result: {+ status: result.status,+ output: truncateMiddle(+ result.output,+ AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,+ ),+ settledAt: result.settledAt ?? new Date(envelope.event.occurredAt),+ },+ })) === true;+ } finally {+ await releaseProjection();+ }+ if (restored) return resolve(envelope, context);+ throw executionError('The background result receipt is being reconstructed.', {+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: '1',+ });+ }+ if (+ projection.status === 'claimed' &&+ projection.claim?.kind === 'manual' &&+ projection.claim.generationId != null+ ) {+ await recoverDeadClaim?.({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ claimId: projection.claim.claimId,+ kind: 'manual',+ generationId: projection.claim.generationId,+ onlyIfUnreconciled: true,+ });+ }+ if (projection.status === 'claimed')+ throw executionError('The background projection is being reconciled.', {+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: '1',+ });++ const producer = await methods.getAgentTriggerDeliveryProducerLease({+ deliveryKey: context.idempotencyKey,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ now: new Date(),+ });+ if (producer.status === 'expired') {+ throw executionError('The process-local background tool executor was lost.', {+ code: 'BACKGROUND_TOOL_PRODUCER_LOST',+ retryable: false,+ });+ }+ }+ throw executionError('The background result batch is not durable yet.', {+ code: 'BACKGROUND_TOOL_RESULT_NOT_READY',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: waitingRetry(envelope.receivedAt),+ });+ }+ if (batch?.status === 'acquired') {+ const owned = { ...batchScope, batchId: batch.batchId };+ const dispatchId = randomUUID();+ let dispatchBegan = false;+ const release = async (): Promise<boolean> => {+ const admitted = await getGenerationAdmissionEvidence?.(+ userId,+ context.idempotencyKey,+ envelope.target.conversationId,+ envelope.target.conversationId,+ );+ if (admitted != null) {+ await methods.confirmAgentBackgroundToolResultBatch?.(owned);+ return false;+ }+ // Absence is not proof. The storage CAS permits only a first definite+ // failure or a never-dispatched plan, never a retry of an ambiguous POST.+ const released = await methods.releaseAgentBackgroundToolResultClaims?.({+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ userId,+ conversationId: envelope.target.conversationId,+ parentMessageId: envelope.target.parentMessageId,+ claimId: context.idempotencyKey,+ batchId: batch.batchId,+ ...(dispatchBegan && { dispatchId }),+ ...(context.deliveryClaimToken != null && {+ deliveryClaimToken: context.deliveryClaimToken,+ }),+ });+ return released === true;+ };+ // A receipt may precede its projection. Reconcile every member, not just+ // the root, against a concurrent manual poll or late message persistence.+ const projections = await Promise.all(+ batch.results.map((result) =>+ methods.claimBackgroundToolResults({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ taskId: result.taskId,+ agentId: envelope.target.agentId,+ kind: 'wakeup',+ claimId: context.idempotencyKey,+ batchId: batch.batchId,+ limit: 1,+ }),+ ),+ );+ if (+ projections.some(+ (projection) =>+ projection.status === 'claimed' || projection.status === 'outcome_unknown',+ )+ ) {+ for (let index = 0; index < projections.length; index++) {+ const stale = projections[index];+ if (stale?.status !== 'claimed' || stale.claim?.kind !== 'wakeup') continue;+ const taskId = batch.results[index]!.taskId;+ const receipt = await methods.getAgentBackgroundToolResultClaim?.({+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ userId,+ conversationId: envelope.target.conversationId,+ parentMessageId: envelope.target.parentMessageId,+ taskId,+ });+ if (receipt?.claimId !== context.idempotencyKey || receipt.batchId !== batch.batchId)+ continue;+ await methods.releaseBackgroundToolResultClaims({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ taskIds: [taskId],+ kind: 'wakeup',+ claimId: stale.claim.claimId,+ ...(stale.claim.batchId != null && { batchId: stale.claim.batchId }),+ allowMissingMessage: true,+ });+ throw executionError('A stale background projection is being reconciled.', {+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: '1',+ });+ }+ const released = await release();+ const rootProjection =+ projections[batch.results.findIndex((result) => result.taskId === registration.taskId)];+ if (+ released &&+ rootProjection?.status === 'claimed' &&+ rootProjection.claim?.kind === 'manual' &&+ rootProjection.claim.receiptReconciled === true+ ) {+ return { status: 'settled' };+ }+ if (released && recoverDeadClaim != null) {+ await Promise.all(+ projections.map(async (projection) => {+ if (+ projection.status !== 'claimed' ||+ projection.claim?.kind !== 'manual' ||+ projection.claim.receiptReconciled === true ||+ projection.claim.generationId == null+ )+ return;+ await recoverDeadClaim({+ userId,+ conversationId: envelope.target.conversationId,+ messageId: envelope.target.parentMessageId,+ claimId: projection.claim.claimId,+ kind: 'manual',+ generationId: projection.claim.generationId,+ onlyIfUnreconciled: true,+ });+ }),+ );+ }+ throw executionError('Background result ownership is being reconciled.', {+ code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: '1',+ });+ }+ return {+ status: 'ready',+ parentMessageId,+ ...(parent.codeApprovalMode != null && { codeApprovalMode: parent.codeApprovalMode }),+ input: buildWakeupInput(batch.results),+ releaseOnDefiniteFailure: async () => {+ await release();+ },+ beginDispatch: async () => {+ dispatchBegan = true;+ if (+ !(await methods.beginAgentBackgroundToolResultBatchDispatch?.({ ...owned, dispatchId }))+ ) {+ throw executionError('The background batch preparation was superseded.', {+ code: 'BACKGROUND_TOOL_BATCH_SUPERSEDED',+ retryable: true,+ deferWithoutAttempt: true,+ });+ }+ },+ settleOnAdmission: async (result) => {+ if (+ result.status === 'settled' &&+ getGenerationAdmissionEvidence != null &&+ (await getGenerationAdmissionEvidence(+ userId,+ context.idempotencyKey,+ envelope.target.conversationId,+ envelope.target.conversationId,+ )) == null+ ) {+ throw new Error('A recovery fence does not prove background admission');+ }+ if (!(await methods.confirmAgentBackgroundToolResultBatch?.(owned))) {+ throw new Error('Background result batch admission was not confirmed');+ }+ },+ };+ }const claim = await methods.claimBackgroundToolResults({userId,conversationId: envelope.target.conversationId,@@ -545,6 +869,7 @@ export function createBackgroundToolCompletionWakeupResolver({deferWithoutAttempt: true,});};+ return resolve;}/** Lists and discards a conversation's undelivered background completions from the@@ -652,6 +977,7 @@ export function createBackgroundToolCompletionWakeupHandler(renewProducerLease: RenewBackgroundToolCompletionProducerLease,persistResult?: PersistBackgroundToolCompletionResult,expedite?: (deliveryKey: string) => void,+ getReceiptBatchingEnabled?: () => boolean,): (registration: BackgroundToolWakeupRegistration,) => Promise<BackgroundToolWakeupAdmission | false> {@@ -695,7 +1021,10 @@ export function createBackgroundToolCompletionWakeupHandler(availableAt: new Date(Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,),- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability:+ getReceiptBatchingEnabled?.() === true+ ? AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3+ : AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),});return {@@ -735,8 +1064,22 @@ export function createBackgroundToolDeadClaimRecovery(claimId: string;}) => Promise<'fenced' | 'started' | 'unavailable'>,releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],+ batchRecovery?: Pick<+ AgentTriggerDeliveryMethods,+ 'getAgentBackgroundToolResultBatch' | 'confirmAgentBackgroundToolResultBatch'+ >,+ getAdmissionEvidence?: BackgroundToolCompletionWakeupResolverDeps['getGenerationAdmissionEvidence'],): BackgroundToolDeadClaimRecovery {- return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {+ return async ({+ userId,+ conversationId,+ messageId,+ claimId,+ kind,+ generationId,+ batchId,+ onlyIfUnreconciled,+ }) => {if (kind === 'manual') {if (generationId == null || generationId.length === 0) {return false;@@ -753,8 +1096,41 @@ export function createBackgroundToolDeadClaimRecovery(messageId,kind: 'manual',claimId,+ ...(onlyIfUnreconciled === true && { onlyIfUnreconciled }),});}+ const owner = {+ deliveryKey: claimId,+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ userId,+ conversationId,+ parentMessageId: messageId,+ };+ const batch = await batchRecovery?.getAgentBackgroundToolResultBatch(owner);+ if (batchId != null && batch != null && batch.batchId !== batchId) return false;+ if (batch?.appliedAt != null) {+ await batchRecovery?.confirmAgentBackgroundToolResultBatch({+ ...owner,+ batchId: batch.batchId,+ });+ return false;+ }+ const confirmAdmission = async (): Promise<boolean> => {+ const evidence = await getAdmissionEvidence?.(+ userId,+ claimId,+ conversationId,+ conversationId,+ );+ if (evidence == null) return false;+ if (batch != null)+ await batchRecovery?.confirmAgentBackgroundToolResultBatch({+ ...owner,+ batchId: batch.batchId,+ });+ return true;+ };+ if (batch != null && (await confirmAdmission())) return false;const claimGenerationIsActive = async (): Promise<boolean> => {const generation = await getGenerationJob(conversationId);return (@@ -778,30 +1154,39 @@ export function createBackgroundToolDeadClaimRecovery(* installs a started tombstone that invalidates a delayed creator's token,* or observes that job creation already won. */const generationFence = await fenceGenerationClaim({ userId, conversationId, claimId });- if (generationFence === 'unavailable') {+ if (generationFence === 'unavailable') return false;+ if (generationFence === 'started' && batch != null) {+ // The native claim CAS proves admission even when a custom store cannot+ // read historical claims or the live generation has been cleaned up.+ await batchRecovery?.confirmAgentBackgroundToolResultBatch({+ ...owner,+ batchId: batch.batchId,+ });return false;}if (await claimGenerationIsActive()) {return false;}- const released = await releaseClaims({+ const receiptBatchId = batch?.batchId ?? batchId;+ const receiptReleased = await releaseReceiptClaims?.({+ sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,+ userId,+ conversationId,+ parentMessageId: messageId,+ claimId,+ ...(receiptBatchId != null && { batchId: receiptBatchId }),+ recoveryFenced: true,+ });+ if (receiptReleased === false) return false;+ if (batch != null) return receiptReleased === true;+ return releaseClaims({userId,conversationId,messageId,+ allowMissingMessage: true,kind: 'wakeup',claimId,+ ...(receiptBatchId != null && { batchId: receiptBatchId }),});- if (!released) {- return false;- }- return (- (await releaseReceiptClaims?.({- sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,- userId,- conversationId,- parentMessageId: messageId,- claimId,- })) ?? true- );};}diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.tsindex 6eac3eedb1..ce75ddac9f 100644--- a/packages/api/src/agents/handlers.background.spec.ts+++ b/packages/api/src/agents/handlers.background.spec.ts@@ -457,6 +457,59 @@ describe('createToolExecuteHandler — background tool calls', () => {expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);});+ it('copies committed manual reconciliation into the final background projection retry', async () => {+ let finishPersistence: (persisted: boolean) => void = () => undefined;+ let resolveProjection: (() => { resultClaim?: { receiptReconciled?: true } }) | undefined;+ const tool = makeSearchTool({ calls: 0 });+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: {+ preregister: async () => ({+ renew: async () => true,+ retire: async () => true,+ persistResult: async () => true,+ }),+ claim: async () => ({ status: 'not_ready' }),+ persist: (params) => {+ resolveProjection = params.resolveBackgroundTask;+ return new Promise<boolean>((resolve) => {+ finishPersistence = resolve;+ });+ },+ },+ });+ const [dispatch] = await runBatch(handler, {+ toolCalls: [+ {+ id: 'manual-marker-dispatch',+ stepId: 'manual-marker-step',+ name: tool.name,+ args: { q: 'marker', run_in_background: true },+ },+ ],+ agentId: 'agent_parent_1',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'manual-marker-convo', run_id: 'manual-marker-parent' },+ });+ await flushMicrotasks();+ const taskId = JSON.parse(dispatch.content).background_task_id as string;+ try {+ expect(resolveProjection?.().resultClaim).toBeUndefined();+ expect(+ backgroundTaskRegistry.claimResult('exec_user', 'manual-marker-convo', taskId, {+ kind: 'manual',+ claimId: 'manual-marker-poll',+ receiptReconciled: true,+ generationId: 'manual-marker-generation',+ }),+ ).toBe('acquired');+ expect(resolveProjection?.().resultClaim).toMatchObject({ receiptReconciled: true });+ } finally {+ finishPersistence(true);+ await flushMicrotasks();+ }+ });+it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => {let finishPersistence: ((persisted: boolean) => void) | undefined;const persist = jest.fn(diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.tsindex 670296a101..bbb1643d7c 100644--- a/packages/api/src/agents/handlers.ts+++ b/packages/api/src/agents/handlers.ts@@ -6446,6 +6446,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandkind: current.resultClaim.kind,claimId: current.resultClaim.claimId,claimedAt: new Date(current.resultClaim.claimedAt),+ ...(current.resultClaim.receiptReconciled === true && {+ receiptReconciled: true,+ }),...(current.resultClaim.generationId == null? {}: { generationId: current.resultClaim.generationId }),diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.tsindex 63cd918861..38e257df91 100644--- a/packages/api/src/agents/harvest.ts+++ b/packages/api/src/agents/harvest.ts@@ -51,6 +51,7 @@ export interface BackgroundToolResultState {claimId: string;claimedAt: Date;generationId?: string;+ receiptReconciled?: true;};}diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.mdindex 5fe25e09aa..3d266ce654 100644--- a/packages/api/src/agents/triggers/README.md+++ b/packages/api/src/agents/triggers/README.md@@ -269,3 +269,75 @@ event whose individual timing or acknowledgment is actionable. `fire`, `steer`,`continue` deliveries reject the option instead of silently weakening their semantics. Deliverieswith `expectedAction` also reject coalescing because one generation cannot prove several distinctaction fences.++## Background receipt batches++Ordinary background tool and code completions use `background_tool_completion_batch_v3`.+Older workers cannot claim these rows; v2 receipts retain task-local delivery. Subagent+completions do not join receipt batches. The existing `completionResultBatchSize` bounds+selection, with no collection window. Scope is owner, tenant, conversation, launching+parent message, and target agent. Results launched on different parent messages remain+separate even when their branches later converge.++The root persists candidate keys before per-result claims, then freezes only successful+claims in `backgroundToolResultBatch.members`. Retries use that same list. Each member's+message projection is reconciled before dispatch. A first definite rejection releases the unadmitted plan. Once a handoff may have been+admitted, later transport failures retain frozen ownership and the same idempotency key. Admission proof is stored before followers settle, and copied to each+receipt so root retention cannot strand a follower.++Followers defer without consuming attempts. A dead or retired root is recovered through+the generation-admission fence before claims are released. Losing collectors release+claims they cannot dispatch. Concurrent roots can still split the ready set; batching is+opportunistic, not one-turn-per-conversation election.++`triggerDelivery.spec.ts` includes a real-Mongo collecting barrier, crash/lost-reply+injection, rolling-upgrade isolation, and an 8-conversation × 4-result storage stress+harness. Its latency measures receipt admission, not model turns or deployment latency.++Receipt batching defaults off. Deploy compatible receipt and manual-poll consumers on+**every replica**, then enable:++```yaml+endpoints:+ agents:+ backgroundTasks:+ completionReceiptBatching: true+```++Older manual-poll workers cannot read v3 ownership. Queue capability fencing alone does+not protect that path. Disabling batching stops new v3 production, not existing+ownership. Do not downgrade poll consumers while v3 receipts remain, including delivered+receipts. A successor lease resumes interrupted cleanup before dispatch.++Every plan, receipt claim, and message claim carries a physical batch identity. Cleanup+matches that identity; its final plan deletion also matches a release identity and the+queue lease. A durable dispatch counter prevents a definite retry failure from releasing+an earlier ambiguous handoff. Native recovery fences distinguish unadmitted tombstones+from actual generation admission. Proven admissions settle followers, never re-present+results.++Retired owners do not expire while a batch still needs claim release or per-receipt+admission proof. Cleanup restores the ordinary retention deadline only after every+member is safe without the owner. Retirement also closes publication and dispatch.++A failed receipt write is repaired from its durable terminal message projection.+Manual polling marks receipt reconciliation before automatic delivery can settle that+handoff; speculative manual ownership only defers the automatic delivery.++Manual confirmation resolves a lost reply by reading the exact committed claim.+Failed confirmation rolls back only unreconciled ownership. If rollback is unavailable,+automatic delivery invokes manual-generation recovery after that owner is no longer active.+Its release CAS excludes committed manual handoffs, including confirmation after a stale+snapshot. Explicit manual recovery keeps its existing generation-fenced contract.++Manual reconciliation survives registry restoration and final response persistence.+Collecting plans acquire the root before siblings, so empty retirement can clear an+unowned plan without orphaning claims. Native `started` recovery fences confirm admission+even when a custom store cannot read historical claims after job cleanup.++Exact durable replays read the current claim without rewriting its handoff proof.+Applied followers finish interrupted owner proof copying before they settle, including+retired roots removed from the delivery queue.++Manual polls carry the observed physical batch ID through recovery. A released owner’s+late projection is cleared by that exact ID; successor epochs and manual claims remain owned.diff --git a/packages/api/src/agents/triggers/dispatch.ts b/packages/api/src/agents/triggers/dispatch.tsindex 09d7607eca..99fe88e14e 100644--- a/packages/api/src/agents/triggers/dispatch.ts+++ b/packages/api/src/agents/triggers/dispatch.ts@@ -11,6 +11,9 @@ export interface AgentTriggerDispatchContext {/** Durable delivery attempt metadata, when dispatched by the queue engine. */attempt?: number;maxAttempts?: number;+ /** Exact queue lease that owns this preparation. */+ deliveryClaimToken?: string;+ requiredWorkerCapability?: string;signal?: AbortSignal;}@@ -44,7 +47,13 @@ export interface AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerRexport function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(envelope: unknown,handlers: AgentTriggerDispatchHandlers<FireResult, ContinueResult, SteerResult>,- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },+ options?: {+ signal?: AbortSignal;+ attempt?: number;+ maxAttempts?: number;+ deliveryClaimToken?: string;+ requiredWorkerCapability?: string;+ },): Promise<ContinueResult | FireResult | SteerResult> {let normalized: AgentTriggerEnvelope;try {@@ -56,6 +65,10 @@ export function dispatchAgentTrigger<FireResult, ContinueResult, SteerResult>(idempotencyKey: getAgentTriggerIdempotencyKey(normalized),...(options?.attempt != null && { attempt: options.attempt }),...(options?.maxAttempts != null && { maxAttempts: options.maxAttempts }),+ ...(options?.deliveryClaimToken != null && { deliveryClaimToken: options.deliveryClaimToken }),+ ...(options?.requiredWorkerCapability != null && {+ requiredWorkerCapability: options.requiredWorkerCapability,+ }),...(options?.signal != null && { signal: options.signal }),};if (normalized.mode === 'fire') {diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.tsindex 697834d4f8..7c151f1026 100644--- a/packages/api/src/agents/triggers/engine.spec.ts+++ b/packages/api/src/agents/triggers/engine.spec.ts@@ -74,7 +74,12 @@ describe('createAgentTriggerDeliveryEngine', () => {expect(dispatch).toHaveBeenCalledWith({ version: 1 },- { signal: expect.any(AbortSignal), attempt: 1, maxAttempts: 8 },+ {+ signal: expect.any(AbortSignal),+ attempt: 1,+ maxAttempts: 8,+ deliveryClaimToken: 'claim-1',+ },);expect(store.beginAttempt).toHaveBeenCalledWith({id: 'delivery-row-1',diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.tsindex e6d6f9e8fc..7f1414ba22 100644--- a/packages/api/src/agents/triggers/engine.ts+++ b/packages/api/src/agents/triggers/engine.ts@@ -92,6 +92,7 @@ export interface AgentTriggerDeliveryRecord {orderingKey: string;laneSequence: number;envelope: unknown;+ requiredWorkerCapability?: string;status: AgentTriggerDeliveryStatus;attempts: number;availableAt: Date;@@ -204,7 +205,13 @@ export interface AgentTriggerDeliveryEngineDeps {store: AgentTriggerDeliveryStore;dispatch: (envelope: unknown,- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },+ options?: {+ signal?: AbortSignal;+ attempt?: number;+ maxAttempts?: number;+ deliveryClaimToken?: string;+ requiredWorkerCapability?: string;+ },) => Promise<AgentTriggerExecutionResult>;/** Source-owned terminalization must commit before its delivery can become* dead, including recovery after a crash that exhausted the attempt budget. */@@ -489,6 +496,10 @@ export function createAgentTriggerDeliveryEngine(signal: controller.signal,attempt,maxAttempts,+ deliveryClaimToken: delivery.claimToken,+ ...(delivery.requiredWorkerCapability != null && {+ requiredWorkerCapability: delivery.requiredWorkerCapability,+ }),});} catch (error) {const attemptedAt = now();diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.tsindex 6bf30f70b8..30c6eb818e 100644--- a/packages/api/src/agents/triggers/host.ts+++ b/packages/api/src/agents/triggers/host.ts@@ -70,6 +70,8 @@ export type AgentTriggerContinuePreparation =releaseOnDefiniteFailure?: (error?: AgentTriggerExecutionError) => MaybePromise<void>;/** Commits the source handoff after generation admission. Failure is* outcome-ambiguous: the same delivery retries with the same request id. */+ /** Durable possible-handoff marker written immediately before transport. */+ beginDispatch?: () => MaybePromise<void>;settleOnAdmission?: (result: AgentTriggerContinueResult) => MaybePromise<void>;}| { status: 'settled' };@@ -172,7 +174,13 @@ export interface AgentTriggerExecutionHostDeps {export interface AgentTriggerExecutionHost {dispatch: (envelope: unknown,- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },+ options?: {+ signal?: AbortSignal;+ attempt?: number;+ maxAttempts?: number;+ deliveryClaimToken?: string;+ requiredWorkerCapability?: string;+ },) => Promise<AgentTriggerExecutionResult>;}@@ -636,6 +644,7 @@ async function startRun(const url = mode === 'fire' ? fireUrl(baseUrl) : continueUrl(baseUrl);const fetcher: AgentTriggerFetch = deps.fetch ?? globalThis.fetch;let response: Response;+ await readyPreparation?.beginDispatch?.();try {response = await fetcher(url, {method: 'POST',diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsindex c62b09e8ce..2f0b9fb1d8 100644--- a/packages/api/src/agents/triggers/service.delivery.spec.ts+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -1,5 +1,6 @@import {AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,@@ -185,6 +186,7 @@ describe('durable agent trigger service', () => {expect.objectContaining({workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,@@ -211,6 +213,14 @@ describe('durable agent trigger service', () => {await service.stop();});+ it('keeps receipt batching disabled until the deployment gate is explicitly enabled', async () => {+ const service = createAgentTriggerService({ mintToken: () => 'test' });+ expect(service.getBackgroundCompletionReceiptBatching()).toBe(false);+ await service.initialize({ address: '127.0.0.1', completionReceiptBatching: true });+ expect(service.getBackgroundCompletionReceiptBatching()).toBe(true);+ await service.stop();+ });+describe('waiting completion deliveries', () => {const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };const flush = () => new Promise((resolve) => setImmediate(resolve));@@ -407,6 +417,7 @@ describe('durable agent trigger service', () => {expect.objectContaining({workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsindex f2786ed860..1dc249251c 100644--- a/packages/api/src/agents/triggers/service.ts+++ b/packages/api/src/agents/triggers/service.ts@@ -1,5 +1,6 @@import {AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,@@ -50,6 +51,7 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;export interface AgentTriggerServiceOptions {completionResultBatchSize?: number;+ completionReceiptBatching?: boolean;address?: BoundAddress | string | null;idlePolling?: {queuedTurnMaxIntervalMs?: number;@@ -230,6 +232,7 @@ export interface AgentTriggerService {input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;getBackgroundCompletionResultBatchSize: () => number;+ getBackgroundCompletionReceiptBatching: () => boolean;/** Longest a waiting completion delivery re-checks readiness. */getCompletionWaitMaxIntervalMs: () => number;/** Best effort: moves waiting completion deliveries forward after what they wait on changed. */@@ -271,6 +274,7 @@ function createDeliveryStore(...input,workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,@@ -352,6 +356,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}let boundOrigin: string | undefined;let backgroundCompletionResultBatchSize = 8;+ let backgroundCompletionReceiptBatching = false;let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;let deliveryEngine: AgentTriggerDeliveryEngine | undefined;let initializePromise: Promise<void> | undefined;@@ -410,7 +415,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Aconst dispatchForActivePrincipal = async (envelope: unknown,- options?: { signal?: AbortSignal; attempt?: number; maxAttempts?: number },+ options?: {+ signal?: AbortSignal;+ attempt?: number;+ maxAttempts?: number;+ deliveryClaimToken?: string;+ requiredWorkerCapability?: string;+ },): Promise<AgentTriggerExecutionResult> => {const parsed = parseAgentTriggerEnvelope(envelope);await requireActivePrincipal(parsed.principal.userId);@@ -619,6 +630,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Areturn {initialize: (options = {}) => {backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;+ backgroundCompletionReceiptBatching = options.completionReceiptBatching === true;completionWaitMaxIntervalMs =options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;@@ -809,6 +821,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Areturn getClaim == null ? null : getClaim(input);}),getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,+ getBackgroundCompletionReceiptBatching: () => backgroundCompletionReceiptBatching,getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,expediteCompletionWakeups: (input) => expediteCompletions(input),releaseBackgroundToolResultClaims: (input) =>diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 40869e850f..3d28faabdc 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -270,7 +270,8 @@ function claimsMirrorExactly(left: TokenIdempotencyClaim, right: TokenIdempotencleft.claimToken === right.claimToken &&left.previousClaimToken === right.previousClaimToken &&left.generationProtocolVersion === right.generationProtocolVersion &&- left.startedAt === right.startedAt+ left.startedAt === right.startedAt &&+ left.recoveryFence === right.recoveryFence);}@@ -3339,7 +3340,7 @@ class GenerationJobManagerClass {}const normalized = normalizeTokenClaim(claim, 'admission evidence');assertClaimMatchesRequest(normalized, streamId, conversationId);- if (normalized.startedAt == null) {+ if (normalized.startedAt == null || normalized.recoveryFence === true) {return null;}return {@@ -3380,14 +3381,14 @@ class GenerationJobManagerClass {return 'unavailable';}if (observed.existing.startedAt != null) {- return 'started';+ return observed.existing.recoveryFence === true ? 'fenced' : 'started';}let owned = observed;if (!observed.claimed) {owned = await this.takeoverGeneration(userId, clientRequestId, streamId, observed.existing);if (owned.existing?.startedAt != null) {- return 'started';+ return owned.existing.recoveryFence === true ? 'fenced' : 'started';}if (!owned.claimed || owned.existing == null) {return 'unavailable';@@ -3396,7 +3397,7 @@ class GenerationJobManagerClass {const claim = normalizeTokenClaim(owned.existing, 'background completion recovery fence');if (claim.startedAt != null) {- return 'started';+ return claim.recoveryFence === true ? 'fenced' : 'started';}await this.tombstoneObservedGenerationClaim(userId,@@ -3405,6 +3406,7 @@ class GenerationJobManagerClass {claim,Date.now(),claim.generationProtocolVersion === 2 ? 2 : 1,+ true,);return 'fenced';}@@ -3477,11 +3479,13 @@ class GenerationJobManagerClass {claim: TokenIdempotencyClaim,createdAt: number,generationProtocolVersion: 1 | 2,+ recoveryFence?: true,): Promise<TokenIdempotencyClaim> {const tombstone: TokenIdempotencyClaim = {...claim,startedAt: createdAt,generationProtocolVersion,+ ...(recoveryFence === true && { recoveryFence }),};const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);let markError: unknown;diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.tsindex 1c8578d86d..117d858e30 100644--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts@@ -394,6 +394,13 @@ describe('GenerationJobManager start-generation claim', () => {});});+ it('does not mistake a replayed recovery tombstone for actual admission proof', async () => {+ const args = ['user-1', 'req-recovery-proof', 'stream-proof', 'stream-proof'] as const;+ expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');+ expect(await manager.fenceGenerationClaimForRecovery(...args)).toBe('fenced');+ expect(await manager.getGenerationAdmissionEvidence(...args)).toBeNull();+ });+it('invalidates an unpublished continuation creator before manual recovery', async () => {const original = await manager.claimGeneration('user-1',diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex 5c705d70bb..467e067985 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -714,6 +714,8 @@ export interface IdempotencyClaimValue {* already-started/cleaned generation and can never be taken over as an* abandoned pre-create lease. */startedAt?: number;+ /** Recovery closed an unpublished claim without admitting a generation. */+ recoveryFence?: true;}/** Result of an atomic {@link IJobStore.claimIdempotencyKey} attempt. */diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.tsindex 14fae0dc87..8c06a6a0dc 100644--- a/packages/data-provider/src/config.spec.ts+++ b/packages/data-provider/src/config.spec.ts@@ -1063,6 +1063,13 @@ describe('agent background completion batch config', () => {endpoints: { agents: { backgroundTasks: {} } },});expect(defaults.endpoints?.agents?.backgroundTasks?.completionResultBatchSize).toBe(8);+ expect(defaults.endpoints?.agents?.backgroundTasks?.completionReceiptBatching).toBe(false);+ expect(+ configSchema.parse({+ version: '1.0',+ endpoints: { agents: { backgroundTasks: { completionReceiptBatching: true } } },+ }).endpoints?.agents?.backgroundTasks?.completionReceiptBatching,+ ).toBe(true);for (const completionResultBatchSize of [0, 17, 1.5]) {expect(@@ -1171,6 +1178,7 @@ describe('agent background task config', () => {}expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({completionResultBatchSize: 8,+ completionReceiptBatching: false,completionWakeups: true,completionResultMaxChars: 24 * 1024,ordinaryToolCancellation: false,@@ -1190,6 +1198,7 @@ describe('agent background task config', () => {}expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({completionResultBatchSize: 8,+ completionReceiptBatching: false,completionWakeups: false,completionResultMaxChars: 24 * 1024,ordinaryToolCancellation: false,@@ -1209,6 +1218,7 @@ describe('agent background task config', () => {}expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({completionResultBatchSize: 8,+ completionReceiptBatching: false,completionWakeups: true,completionResultMaxChars: 24 * 1024,ordinaryToolCancellation: true,diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex f64b03666c..3ea0043fa2 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -1736,8 +1736,9 @@ export const agentsEndpointSchema = baseEndpointSchema.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX).optional().default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),- /** Maximum message-backed sibling results in one continuation.- * Independent receipts retain task-local delivery ownership. */+ /** Enable only after every replica has compatible receipt/poll consumers. */+ completionReceiptBatching: z.boolean().optional().default(false),+ /** Maximum compatible sibling results in one continuation. */completionResultBatchSize: z.number().int().min(1).max(16).optional().default(8),/** Cooperative cancellation for process-local ordinary tools. Off* by default so existing deployments opt into the new control. */diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.tsindex e2f7e93ce9..6afad13b80 100644--- a/packages/data-schemas/src/index.ts+++ b/packages/data-schemas/src/index.ts@@ -76,6 +76,7 @@ export {export {AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.tsindex 825fe5a15d..c5417b8d7a 100644--- a/packages/data-schemas/src/methods/index.ts+++ b/packages/data-schemas/src/methods/index.ts@@ -347,6 +347,7 @@ export function createMethods(const agentQueuedTurnMethods = createAgentQueuedTurnMethods(mongoose);const agentTriggerDeliveryMethods = createAgentTriggerDeliveryMethods(mongoose, {+ releaseBatchProjections: messageMethods.releaseBackgroundToolResultClaims,purgeQueuedTurnsForUser: (user) =>agentQueuedTurnMethods.deleteAllAgentQueuedTurnsForUser({user: typeof user === 'string' ? new mongoose.Types.ObjectId(user) : user,diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.tsindex 5e029e423a..3b87bbf478 100644--- a/packages/data-schemas/src/methods/message.spec.ts+++ b/packages/data-schemas/src/methods/message.spec.ts@@ -5991,3 +5991,94 @@ describe('Message Operations', () => {});});});++describe('receipt-backed claim release before projection', () => {+ it('allows a missing message only when the receipt owner explicitly opts in', async () => {+ const input = {+ userId: new mongoose.Types.ObjectId().toString(),+ conversationId: 'missing-conversation',+ messageId: 'missing-parent',+ kind: 'wakeup' as const,+ claimId: 'batch-root',+ };+ await expect(releaseBackgroundToolResultClaims(input)).resolves.toBe(false);+ await expect(+ releaseBackgroundToolResultClaims({ ...input, allowMissingMessage: true }),+ ).resolves.toBe(true);+ });+});++describe('receipt batch projection epochs', () => {+ it('does not let delayed cleanup remove a successor claim on the same logical request', async () => {+ await Message.create({+ messageId: 'epoch-parent',+ conversationId: 'epoch-conversation',+ user: 'epoch-owner',+ isCreatedByUser: false,+ unfinished: false,+ content: [+ {+ type: 'tool_call',+ tool_call: {+ id: 'epoch-call',+ output: 'done',+ backgroundTask: {+ taskId: 'epoch-task',+ toolName: 'tool',+ status: 'completed',+ completionWakeup: true,+ completionReceipt: true,+ },+ },+ },+ ],+ });+ const input = {+ userId: 'epoch-owner',+ conversationId: 'epoch-conversation',+ messageId: 'epoch-parent',+ taskId: 'epoch-task',+ kind: 'wakeup' as const,+ claimId: 'logical-root',+ limit: 1,+ };+ expect((await claimBackgroundToolResults({ ...input, batchId: 'old' })).status).toBe(+ 'acquired',+ );+ let pause: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const paused = new Promise<void>((resolve) => {+ pause = resolve;+ });+ const barrier = new Promise<void>((resolve) => {+ resume = resolve;+ });+ const update = Message.findOneAndUpdate.bind(Message);+ const oldCleanup = jest.spyOn(Message, 'findOneAndUpdate').mockImplementationOnce((...args) => {+ const query = update(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ pause();+ await barrier;+ return execute();+ });+ return query;+ });+ const predecessor = releaseBackgroundToolResultClaims({ ...input, batchId: 'old' });+ await paused;+ expect(await releaseBackgroundToolResultClaims({ ...input, batchId: 'old' })).toBe(true);+ expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(+ 'acquired',+ );+ resume();+ expect(await predecessor).toBe(true);+ oldCleanup.mockRestore();+ expect((await claimBackgroundToolResults({ ...input, batchId: 'new' })).status).toBe(+ 'acquired',+ );+ expect(await claimBackgroundToolResults({ ...input, batchId: 'old' })).toMatchObject({+ status: 'claimed',+ claim: { batchId: 'new' },+ });+ });+});diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.tsindex 5d0ec86234..66181b1b7c 100644--- a/packages/data-schemas/src/methods/message.ts+++ b/packages/data-schemas/src/methods/message.ts@@ -567,7 +567,13 @@ export type BackgroundToolResultClaim =| { status: 'outcome_unknown'; toolName: string }| {status: 'claimed';- claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string };+ claim?: {+ kind: 'manual' | 'wakeup';+ claimId: string;+ generationId?: string;+ batchId?: string;+ receiptReconciled?: true;+ };messageId?: string;}| { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string };@@ -846,6 +852,7 @@ export interface MessageMethods {claimId: string;claimedAt: Date;generationId?: string;+ receiptReconciled?: true;};};}): Promise<{ matched: boolean; unfinished: boolean }>;@@ -858,6 +865,8 @@ export interface MessageMethods {agentId?: string;kind: 'manual' | 'wakeup';claimId: string;+ /** Physical receipt-batch identity, independent of its logical request. */+ batchId?: string;/** Response generation that owns this manual result delivery. */generationId?: string;/** Manual owner-process takeover after automatic delivery was retired. */@@ -866,12 +875,24 @@ export interface MessageMethods {/** Includes JSON escaping, delimiters, and empty result fields. */maxMetadataChars?: number;}): Promise<BackgroundToolResultClaim>;+ confirmBackgroundToolResultClaim(params: {+ userId: string;+ conversationId: string;+ messageId: string;+ taskId: string;+ claimId: string;+ }): Promise<boolean>;releaseBackgroundToolResultClaims(params: {userId: string;conversationId: string;messageId: string;/** Omit to release every sibling owned by this exact batch claim. */taskIds?: string[];+ /** Receipt-backed results can precede the message projection. */+ allowMissingMessage?: true;+ /** Manual rollback must not erase a committed receipt handoff. */+ onlyIfUnreconciled?: true;+ batchId?: string;kind: 'manual' | 'wakeup';claimId: string;}): Promise<boolean>;@@ -1983,7 +2004,15 @@ export function createMessageMethods(function readBackgroundToolResultClaim(row: Pick<IMessage, 'content'>,taskId: string,- ): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined {+ ):+ | {+ kind: 'manual' | 'wakeup';+ claimId: string;+ generationId?: string;+ batchId?: string;+ receiptReconciled?: true;+ }+ | undefined {for (const part of row.content ?? []) {if (part == null || typeof part !== 'object' || Array.isArray(part)) {continue;@@ -1993,7 +2022,13 @@ export function createMessageMethods(tool_call?: {backgroundTask?: {taskId?: unknown;- resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown };+ resultClaim?: {+ kind?: unknown;+ claimId?: unknown;+ generationId?: unknown;+ batchId?: unknown;+ receiptReconciled?: unknown;+ };};};}@@ -2010,6 +2045,8 @@ export function createMessageMethods(return {kind: claim.kind,claimId: claim.claimId,+ ...(typeof claim.batchId === 'string' && { batchId: claim.batchId }),+ ...(claim.receiptReconciled === true && { receiptReconciled: true }),...(typeof claim.generationId === 'string' && claim.generationId.length > 0? { generationId: claim.generationId }: {}),@@ -2021,7 +2058,7 @@ export function createMessageMethods(function parseBackgroundToolResults(message: IMessage,- claim: { kind: 'manual' | 'wakeup'; claimId: string },+ claim: { kind: 'manual' | 'wakeup'; claimId: string; batchId?: string },): BackgroundToolResultRecord[] {const results: BackgroundToolResultRecord[] = [];for (const part of message.content ?? []) {@@ -2040,7 +2077,7 @@ export function createMessageMethods(status?: unknown;cancelled?: unknown;settledAt?: unknown;- resultClaim?: { kind?: unknown; claimId?: unknown };+ resultClaim?: { kind?: unknown; claimId?: unknown; batchId?: unknown };};};};@@ -2052,7 +2089,8 @@ export function createMessageMethods(typeof task.toolName !== 'string' ||(task.status !== 'completed' && task.status !== 'error') ||task.resultClaim?.kind !== claim.kind ||- task.resultClaim.claimId !== claim.claimId+ task.resultClaim.claimId !== claim.claimId ||+ task.resultClaim.batchId !== claim.batchId) {continue;}@@ -2133,6 +2171,7 @@ export function createMessageMethods(kind,claimId,generationId,+ batchId,allowUnfinished = false,limit = kind === 'wakeup' ? 8 : 1,maxMetadataChars,@@ -2144,6 +2183,7 @@ export function createMessageMethods(if ((requestedMessageId != null &&(requestedMessageId.length === 0 || requestedMessageId.length > 256)) ||+ (batchId != null && (batchId.length === 0 || batchId.length > 128 || kind !== 'wakeup')) ||taskId.length === 0 ||taskId.length > 256 ||claimId.length === 0 ||@@ -2214,7 +2254,10 @@ export function createMessageMethods(}const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {};const requestedClaim = readBackgroundToolResultClaim(row, taskId);- const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;+ const replaying =+ requestedClaim?.kind === kind &&+ requestedClaim.claimId === claimId &&+ requestedClaim.batchId === batchId;const candidates: string[] = [];const costs = new Map<string, number>();let receiptBacked = false;@@ -2233,8 +2276,13 @@ export function createMessageMethods(}const terminal = task?.status === 'completed' || task?.status === 'error';const wakeupEligible = kind !== 'wakeup' || task?.completionWakeup === true;- const resultClaim = task?.resultClaim as { kind?: unknown; claimId?: unknown } | undefined;- const replay = resultClaim?.kind === kind && resultClaim.claimId === claimId;+ const resultClaim = task?.resultClaim as+ | { kind?: unknown; claimId?: unknown; batchId?: unknown }+ | undefined;+ const replay =+ resultClaim?.kind === kind &&+ resultClaim.claimId === claimId &&+ resultClaim.batchId === batchId;const sameAgent =agentId == null ||partAgentId == null ||@@ -2315,105 +2363,103 @@ export function createMessageMethods(kind,claimId,claimedAt,+ ...(batchId != null && { batchId }),...(kind === 'manual' && requestedGenerationId != null? { generationId: requestedGenerationId }: {}),};- const updated = await Message.findOneAndUpdate(- {- user: userId,- conversationId,- messageId: resolvedMessageId,- ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),- content: {- $elemMatch: {- type: 'tool_call',- 'tool_call.backgroundTask.taskId': taskId,- 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },- ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),- ...(replaying- ? {- 'tool_call.backgroundTask.resultClaim.kind': kind,- 'tool_call.backgroundTask.resultClaim.claimId': claimId,- }- : /** Missing OR stored null: the in-memory claimable scan, the- * claim arrayFilters, and the settle stamp all treat a null- * claim as unclaimed, and the subfield-preserving settle write- * keeps a persisted null a whole-object rewrite used to drop.- * `$exists: false` here would strand such a part as terminal- * but permanently unclaimable. */- { 'tool_call.backgroundTask.resultClaim': null }),- },+ const claimFilter: FilterQuery<IMessage> = {+ user: userId,+ conversationId,+ messageId: resolvedMessageId,+ ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),+ content: {+ $elemMatch: {+ type: 'tool_call',+ 'tool_call.backgroundTask.taskId': taskId,+ 'tool_call.backgroundTask.status': { $in: ['completed', 'error'] },+ ...(kind === 'wakeup' ? { 'tool_call.backgroundTask.completionWakeup': true } : {}),+ ...(replaying+ ? {+ 'tool_call.backgroundTask.resultClaim.kind': kind,+ 'tool_call.backgroundTask.resultClaim.claimId': claimId,+ 'tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,+ }+ : /** Missing OR stored null: the in-memory claimable scan, the+ * claim arrayFilters, and the settle stamp all treat a null+ * claim as unclaimed, and the subfield-preserving settle write+ * keeps a persisted null a whole-object rewrite used to drop.+ * `$exists: false` here would strand such a part as terminal+ * but permanently unclaimable. */+ { 'tool_call.backgroundTask.resultClaim': null }),},- ...(agentId != null- ? {- $expr: {- $anyElementTrue: {- $map: {- input: { $ifNull: ['$content', []] },- as: 'candidate',- in: {- $and: [- { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },- {- $in: [- {- $ifNull: [- {- $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],- },- null,- ],- },- [null, agentId],- ],- },- ],- },+ },+ ...(agentId != null+ ? {+ $expr: {+ $anyElementTrue: {+ $map: {+ input: { $ifNull: ['$content', []] },+ as: 'candidate',+ in: {+ $and: [+ { $eq: ['$$candidate.tool_call.backgroundTask.taskId', taskId] },+ {+ $in: [+ {+ $ifNull: [+ {+ $ifNull: ['$$candidate.agentId', '$$candidate.tool_call.agentId'],+ },+ null,+ ],+ },+ [null, agentId],+ ],+ },+ ],},},},- }- : {}),- },- /** Stamps the claim onto every part this pass admitted. The filtered- * positional operator selects those parts by predicate, so the write- * touches only them instead of re-emitting the whole content array, and- * needs no read-modify-write. Amazon DocumentDB rejects the- * aggregation-pipeline form this replaces. */- { $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },- {- new: true,- projection: { content: 1 },- arrayFilters: [+ },+ }+ : {}),+ };+ const updated = replaying+ ? await Message.findOne(claimFilter).select({ content: 1 }).lean<IMessage | null>()+ : await Message.findOneAndUpdate(+ claimFilter,+ /** Stamps the claim onto every part this pass admitted. The filtered+ * positional operator selects those parts by predicate, so the write+ * touches only them instead of re-emitting the whole content array, and+ * needs no read-modify-write. Amazon DocumentDB rejects the+ * aggregation-pipeline form this replaces. */+ { $set: { 'content.$[part].tool_call.backgroundTask.resultClaim': claimStamp } },{- 'part.type': 'tool_call',- 'part.tool_call.backgroundTask.taskId': { $in: candidates },- 'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },- ...(kind === 'wakeup'- ? { 'part.tool_call.backgroundTask.completionWakeup': true }- : {}),- $and: [+ new: true,+ projection: { content: 1 },+ arrayFilters: [{- /** Unclaimed, or already held by this exact claimant (replay). */- $or: [- { 'part.tool_call.backgroundTask.resultClaim': null },+ 'part.type': 'tool_call',+ 'part.tool_call.backgroundTask.taskId': { $in: candidates },+ 'part.tool_call.backgroundTask.status': { $in: ['completed', 'error'] },+ ...(kind === 'wakeup'+ ? { 'part.tool_call.backgroundTask.completionWakeup': true }+ : {}),+ $and: [{- 'part.tool_call.backgroundTask.resultClaim.kind': kind,- 'part.tool_call.backgroundTask.resultClaim.claimId': claimId,+ 'part.tool_call.backgroundTask.resultClaim': null,},+ ...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),],},- ...(agentId == null ? [] : [agentOwnershipFilter('part.', agentId)]),],},- ],- },- ).lean<IMessage | null>();+ ).lean<IMessage | null>();if (updated == null) {return { status: 'not_ready' };}- const results = parseBackgroundToolResults(updated, { kind, claimId });+ const results = parseBackgroundToolResults(updated, { kind, claimId, batchId });const competingClaim = readBackgroundToolResultClaim(updated, taskId);return results.some((result) => result.taskId === taskId)? { status: 'acquired', results, ...recoveredSource }@@ -2424,21 +2470,82 @@ export function createMessageMethods(};}+ /** Manual ownership is committed only after receipt arbitration completes. */+ async function confirmBackgroundToolResultClaim(+ input: Parameters<MessageMethods['confirmBackgroundToolResultClaim']>[0],+ ): Promise<boolean> {+ const Message = mongoose.models.Message as Model<IMessage>;+ const identity = {+ 'tool_call.backgroundTask.taskId': input.taskId,+ 'tool_call.backgroundTask.resultClaim.kind': 'manual',+ 'tool_call.backgroundTask.resultClaim.claimId': input.claimId,+ };+ try {+ const confirmed = await Message.updateOne(+ {+ user: input.userId,+ conversationId: input.conversationId,+ messageId: input.messageId,+ content: { $elemMatch: identity },+ },+ {+ $set: { 'content.$[part].tool_call.backgroundTask.resultClaim.receiptReconciled': true },+ },+ {+ arrayFilters: [+ {+ 'part.tool_call.backgroundTask.taskId': input.taskId,+ 'part.tool_call.backgroundTask.resultClaim.kind': 'manual',+ 'part.tool_call.backgroundTask.resultClaim.claimId': input.claimId,+ },+ ],+ },+ );+ return confirmed.matchedCount === 1;+ } catch (error) {+ // A lost write reply is not an uncommitted handoff. Read only the exact+ // manual claim, never manufacture success from another claimant.+ const committed = await Message.exists({+ user: input.userId,+ conversationId: input.conversationId,+ messageId: input.messageId,+ content: {+ $elemMatch: {+ ...identity,+ 'tool_call.backgroundTask.resultClaim.receiptReconciled': true,+ },+ },+ });+ if (committed != null) return true;+ throw error;+ }+ }+async function releaseBackgroundToolResultClaims({userId,conversationId,messageId,taskIds,+ allowMissingMessage,+ onlyIfUnreconciled,kind,claimId,+ batchId,}: {userId: string;conversationId: string;messageId: string;taskIds?: string[];+ /** Receipt-backed results can precede the message projection. */+ allowMissingMessage?: true;+ /** Manual rollback must not erase a committed receipt handoff. */+ onlyIfUnreconciled?: true;+ batchId?: string;kind: 'manual' | 'wakeup';claimId: string;}): Promise<boolean> {+ if (onlyIfUnreconciled === true && kind !== 'manual')+ throw new TypeError('Unreconciled rollback requires a manual claim');if (taskIds?.length === 0) {return true;}@@ -2460,6 +2567,10 @@ export function createMessageMethods({'part.tool_call.backgroundTask.resultClaim.kind': kind,'part.tool_call.backgroundTask.resultClaim.claimId': claimId,+ 'part.tool_call.backgroundTask.resultClaim.batchId': batchId ?? null,+ ...(onlyIfUnreconciled === true && {+ 'part.tool_call.backgroundTask.resultClaim.receiptReconciled': { $ne: true },+ }),...(taskIds == null? {}: { 'part.tool_call.backgroundTask.taskId': { $in: taskIds } }),@@ -2474,9 +2585,13 @@ export function createMessageMethods(messageId,content: { $not: { $type: 'array' } },});- return arraylessRow != null;+ if (arraylessRow != null) return true;+ return (+ allowMissingMessage === true &&+ (await Message.exists({ user: userId, conversationId, messageId })) == null+ );}- const remaining = parseBackgroundToolResults(updated, { kind, claimId });+ const remaining = parseBackgroundToolResults(updated, { kind, claimId, batchId });return taskIds == null? remaining.length === 0: !remaining.some((result) => taskIds.includes(result.taskId));@@ -4249,6 +4364,7 @@ export function createMessageMethods(updateMessageText,updateToolCallResult,claimBackgroundToolResults,+ confirmBackgroundToolResultClaim,releaseBackgroundToolResultClaims,updateMessage,recordSubagentTaskControlReceipt,diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.tsindex c49c46ea22..51e878d37e 100644--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts@@ -10,6 +10,7 @@ import type {import type { IUser } from '~/types/user';import {AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2,@@ -28,6 +29,8 @@ import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequenimport { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';import { createConversationModel } from '../models/convo';+import { createMessageModel } from '../models/message';+import { createMessageMethods } from './message';import { createUserModel } from '../models/user';jest.mock('~/config/winston', () => ({@@ -56,8 +59,11 @@ beforeAll(async () => {UserPurge = createAgentTriggerUserPurgeModel(mongoose);User = createUserModel(mongoose);createConversationModel(mongoose);+ createMessageModel(mongoose);await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);- methods = createAgentTriggerDeliveryMethods(mongoose);+ methods = createAgentTriggerDeliveryMethods(mongoose, {+ releaseBatchProjections: createMessageMethods(mongoose).releaseBackgroundToolResultClaims,+ });}, DB_SETUP_TIMEOUT_MS);afterAll(async () => {@@ -79,6 +85,14 @@ afterAll(() => {setAgentEventActorReceiptMetricObserver();});+async function confirmBatch(+ input: Parameters<typeof methods.getAgentBackgroundToolResultBatch>[0],+) {+ const batch = await methods.getAgentBackgroundToolResultBatch(input);+ if (batch == null) return false;+ return methods.confirmAgentBackgroundToolResultBatch({ ...input, batchId: batch.batchId });+}+function enqueueInput(overrides: Partial<Parameters<typeof methods.enqueueAgentTriggerDelivery>[0]> = {},) {@@ -5554,3 +5568,1079 @@ describe('agent trigger delivery methods', () => {expect(await UserPurge.countDocuments({ _id: user })).toBe(1);});});++describe('background result receipt batches', () => {+ const capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3;+ const sourceId = 'background-tool-completion';+ let user: mongoose.Types.ObjectId;+ beforeEach(() => {+ user = new mongoose.Types.ObjectId();+ });++ async function ready(+ key: string,+ overrides: Partial<Parameters<typeof methods.enqueueAgentTriggerDelivery>[0]> = {},+ ) {+ await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ deliveryKey: key,+ orderingKey: `lane-${key}`,+ user,+ tenantId: 'tenant-1',+ requiredWorkerCapability: capability,+ envelope: {+ event: {+ type: 'background-tool.completion',+ source: { id: sourceId, type: 'internal' },+ payload: { taskId: key, toolCallId: `call-${key}`, toolName: 'tool' },+ },+ target: { conversationId: 'conversation', parentMessageId: 'parent', agentId: 'agent' },+ },+ ...overrides,+ }),+ );+ await methods.persistAgentBackgroundToolResult({+ deliveryKey: key,+ sourceId,+ result: { status: 'completed', output: `output-${key}`, settledAt: START },+ });+ }++ function input(deliveryKey: string) {+ return {+ deliveryKey,+ sourceId,+ userId: user.toString(),+ tenantId: 'tenant-1',+ conversationId: 'conversation',+ parentMessageId: 'parent',+ agentId: 'agent',+ limit: 8,+ maxMetadataChars: 16000,+ };+ }++ it('freezes ready membership, defers followers, and confirms every receipt before settlement', async () => {+ await ready('one');+ await ready('two');+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ expect(batch).toMatchObject({+ status: 'acquired',+ results: [{ taskId: 'one' }, { taskId: 'two' }],+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'pending',+ });+ await ready('late');+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(batch);+ expect(await confirmBatch(input('one'))).toBe(true);+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'applied',+ });+ await Delivery.deleteOne({ deliveryKey: 'one' });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'applied',+ });+ expect((await methods.claimAgentBackgroundToolResultBatch(input('late'))).status).toBe(+ 'acquired',+ );+ });++ it.each(['claim', 'membership'] as const)(+ 'reconstructs the same candidates after a crash during %s',+ async (phase) => {+ await ready('one');+ await ready('two');+ const updateOne = Delivery.updateOne.bind(Delivery);+ const crash =+ phase === 'claim'+ ? jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {+ throw new Error('crash after candidate write');+ })+ : jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {+ const query = updateOne(...args);+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResultBatch.members'))+ throw new Error('crash after result claims');+ return query;+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(+ 'crash',+ );+ crash.mockRestore();+ await ready('late');+ const retry = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ expect(retry).toMatchObject({+ status: 'acquired',+ results: [{ taskId: 'one' }, { taskId: 'two' }],+ });+ expect(JSON.stringify(retry)).not.toContain('late');+ },+ );++ it('reads back frozen membership after a lost write reply', async () => {+ await ready('one');+ await ready('two');+ const updateOne = Delivery.updateOne.bind(Delivery);+ const lostReply = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {+ const query = updateOne(...args);+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResultBatch.members')) {+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ await execute();+ throw new Error('lost membership reply');+ });+ }+ return query;+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(+ 'lost membership reply',+ );+ lostReply.mockRestore();+ await ready('late');+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({+ status: 'acquired',+ results: [{ taskId: 'one' }, { taskId: 'two' }],+ });+ });++ it('returns only successful per-result claims when another root wins part of the selection', async () => {+ await ready('one');+ await ready('two');+ await ready('three');+ const updateMany = Delivery.updateMany.bind(Delivery);+ const partial = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce((...args) => {+ return updateMany({ ...args[0], deliveryKey: { $in: ['one', 'three'] } }, args[1], args[2]);+ });+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({+ status: 'acquired',+ results: [{ taskId: 'one' }, { taskId: 'three' }],+ });+ partial.mockRestore();+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(+ 'acquired',+ );+ });++ it('releases every member and clears the frozen plan after definite rejection', async () => {+ await ready('one');+ await ready('two');+ await methods.claimAgentBackgroundToolResultBatch(input('one'));+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({ ...input('one'), claimId: 'one' }),+ ).toBe(true);+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),+ ).toBeNull();+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(+ 'acquired',+ );+ });++ it.each(['dead', 'retired'] as const)(+ 'keeps followers recoverable when their root is %s',+ async (terminal) => {+ await ready('one');+ await ready('two');+ await methods.claimAgentBackgroundToolResultBatch(input('one'));+ await Delivery.updateOne(+ { deliveryKey: 'one' },+ terminal === 'dead'+ ? { $set: { status: 'leased', capabilityStatus: 'dead' } }+ : { $set: { status: 'succeeded', result: { backgroundToolCompletionRetired: true } } },+ );+ await expect(+ methods.claimAgentBackgroundToolResultBatch(input('two')),+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'recoverable' });+ expect(await Delivery.exists({ deliveryKey: 'two' })).not.toBeNull();+ },+ );++ it('does not mix tenants, principals, branches, agents, sources, or v2 deliveries', async () => {+ await ready('one');+ await ready('v2', {+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ });+ await ready('tenant', { tenantId: 'tenant-2' });+ await ready('user', { user: new mongoose.Types.ObjectId() });+ for (const [key, field, value] of [+ ['branch', 'parentMessageId', 'other-parent'],+ ['agent', 'agentId', 'other-agent'],+ ['conversation', 'conversationId', 'other-conversation'],+ ] as const) {+ await ready(key, {+ envelope: {+ event: {+ type: 'background-tool.completion',+ source: { id: sourceId, type: 'internal' },+ payload: { taskId: key, toolCallId: key, toolName: 'tool' },+ },+ target: {+ conversationId: 'conversation',+ parentMessageId: 'parent',+ agentId: 'agent',+ [field]: value,+ },+ },+ });+ }+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({+ status: 'acquired',+ results: [{ taskId: 'one' }],+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('v2'))).resolves.toEqual({+ status: 'legacy',+ });+ await expect(+ methods.claimAgentBackgroundToolResultBatch({ ...input('tenant'), tenantId: 'tenant-1' }),+ ).resolves.toEqual({ status: 'not_ready' });+ });++ it('prevents v2 workers from acquiring either a root or a follower during a rolling upgrade', async () => {+ await ready('one');+ await ready('two');+ await methods.claimAgentBackgroundToolResultBatch(input('one'));+ await expect(+ methods.claimNextAgentTriggerDelivery({+ workerId: 'old',+ claimToken: 'old-token',+ now: START,+ leaseUntil: new Date(START.getTime() + 60000),+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],+ }),+ ).resolves.toBeNull();+ const next = await methods.claimNextAgentTriggerDelivery({+ workerId: 'new',+ claimToken: 'new-token',+ now: START,+ leaseUntil: new Date(START.getTime() + 60000),+ workerCapabilities: [capability],+ });+ expect(next).not.toBeNull();+ });++ it('resumes an interrupted release under the successor queue lease', async () => {+ await ready('one');+ await ready('two');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'old' } });+ await methods.claimAgentBackgroundToolResultBatch({+ ...input('one'),+ deliveryClaimToken: 'old',+ });+ const updateOne = Delivery.findOneAndUpdate.bind(Delivery);+ const lostReply = jest.spyOn(Delivery, 'findOneAndUpdate').mockImplementationOnce((...args) => {+ const query = updateOne(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ await execute();+ throw new Error('lost release fence reply');+ });+ return query;+ });+ await expect(+ methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ claimId: 'one',+ deliveryClaimToken: 'old',+ }),+ ).rejects.toThrow('lost release fence reply');+ lostReply.mockRestore();+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'new' } });+ expect(+ await methods.claimAgentBackgroundToolResultBatch({+ ...input('one'),+ deliveryClaimToken: 'new',+ }),+ ).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }, { taskId: 'two' }] });+ expect(await confirmBatch(input('one'))).toBe(true);+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'applied',+ });+ });++ it('cannot discard a follower after a root acquired its receipt', async () => {+ await ready('one');+ await ready('two');+ await methods.claimAgentBackgroundToolResultBatch(input('one'));+ await expect(+ methods.retireAgentTriggerDelivery({+ deliveryKey: 'two',+ sourceId,+ settledAt: START,+ reason: 'discarded',+ onlyIfUnclaimed: true,+ requireTransition: true,+ }),+ ).resolves.toBe(false);+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toMatchObject({+ status: 'acquired',+ results: [{ taskId: 'one' }, { taskId: 'two' }],+ });+ });++ it('keeps rollout-default receipts visible to old manual polling after late projection', async () => {+ await ready('one', {+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ });+ await methods.claimAgentBackgroundToolResults({ ...input('one'), claimId: 'one' });+ const oldLookup = await Delivery.findOne({+ user,+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ 'envelope.target.conversationId': 'conversation',+ 'envelope.target.parentMessageId': 'parent',+ 'envelope.event.payload.taskId': 'one',+ })+ .select('+backgroundToolResult')+ .lean();+ expect(oldLookup?.backgroundToolResult?.resultClaim?.claimId).toBe('one');+ });++ it('retains frozen membership across an ambiguous POST and a definitely failed retry', async () => {+ await ready('one');+ await ready('two');+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (batch.status !== 'acquired') throw new Error('Expected acquired batch');+ const owner = { ...input('one'), batchId: batch.batchId };+ await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'first' });+ await ready('late');+ await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'retry' });+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...owner,+ claimId: 'one',+ dispatchId: 'retry',+ }),+ ).toBe(false);+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(batch);+ expect(await methods.confirmAgentBackgroundToolResultBatch(owner)).toBe(true);+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input('late'), taskId: 'late' }),+ ).toBeNull();+ expect((await methods.claimAgentBackgroundToolResultBatch(input('late'))).status).toBe(+ 'acquired',+ );+ });++ it('releases only a proven first handoff failure', async () => {+ await ready('one');+ await ready('two');+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (batch.status !== 'acquired') throw new Error('Expected acquired batch');+ const owner = { ...input('one'), batchId: batch.batchId };+ await methods.beginAgentBackgroundToolResultBatchDispatch({ ...owner, dispatchId: 'first' });+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...owner,+ claimId: 'one',+ dispatchId: 'different-attempt',+ }),+ ).toBe(false);+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...owner,+ claimId: 'one',+ dispatchId: 'first',+ }),+ ).toBe(true);+ });++ it.each(['receipts', 'plan'] as const)(+ 'fences a predecessor already paused before %s cleanup',+ async (phase) => {+ await ready('one');+ await ready('two');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'old' } });+ const original = await methods.claimAgentBackgroundToolResultBatch({+ ...input('one'),+ deliveryClaimToken: 'old',+ });+ if (original.status !== 'acquired') throw new Error('Expected batch');+ let pause: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const paused = new Promise<void>((resolve) => {+ pause = resolve;+ });+ const barrier = new Promise<void>((resolve) => {+ resume = resolve;+ });+ let intercept = true;+ const one = Delivery.updateOne.bind(Delivery);+ const many = Delivery.updateMany.bind(Delivery);+ const block = (query: ReturnType<typeof Delivery.updateOne>) => {+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ pause();+ await barrier;+ return execute();+ });+ return query;+ };+ const spy =+ phase === 'plan'+ ? jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {+ const query = one(...args);+ if (+ intercept &&+ JSON.stringify(query.getUpdate()).includes('"backgroundToolResultBatch":1')+ ) {+ intercept = false;+ return block(query);+ }+ return query;+ })+ : jest.spyOn(Delivery, 'updateMany').mockImplementation((...args) => {+ const query = many(...args);+ if (+ intercept &&+ JSON.stringify(query.getUpdate()).includes('"backgroundToolResult.resultClaim":1')+ ) {+ intercept = false;+ return block(query);+ }+ return query;+ });+ const stale = methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ batchId: original.batchId,+ claimId: 'one',+ deliveryClaimToken: 'old',+ });+ await paused;+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'new' } });+ const successor = await methods.claimAgentBackgroundToolResultBatch({+ ...input('one'),+ deliveryClaimToken: 'new',+ });+ if (successor.status !== 'acquired') throw new Error('Expected successor');+ expect(successor.batchId).not.toBe(original.batchId);+ resume();+ expect(await stale).toBe(false);+ spy.mockRestore();+ expect(+ await methods.confirmAgentBackgroundToolResultBatch({+ ...input('one'),+ batchId: successor.batchId,+ }),+ ).toBe(true);+ await expect(+ methods.claimAgentBackgroundToolResultBatch(input('two')),+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' });+ },+ );++ it('does not let a resumed cleanup snapshot erase a replacement batch without a queue token', async () => {+ await ready('one');+ await ready('two');+ const old = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (old.status !== 'acquired') throw new Error('Expected old batch');+ await Delivery.updateOne(+ { deliveryKey: 'one' },+ { $set: { 'backgroundToolResultBatch.releasing': true } },+ );+ let pause: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const paused = new Promise<void>((resolve) => {+ pause = resolve;+ });+ const barrier = new Promise<void>((resolve) => {+ resume = resolve;+ });+ const update = Delivery.findOneAndUpdate.bind(Delivery);+ const cleanup = jest.spyOn(Delivery, 'findOneAndUpdate').mockImplementationOnce((...args) => {+ const query = update(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ pause();+ await barrier;+ return execute();+ });+ return query;+ });+ const predecessor = methods.claimAgentBackgroundToolResultBatch(input('one'));+ await paused;+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ claimId: 'one',+ batchId: old.batchId,+ }),+ ).toBe(true);+ const current = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (current.status !== 'acquired') throw new Error('Expected replacement batch');+ resume();+ expect(await predecessor).toEqual({ status: 'not_ready' });+ cleanup.mockRestore();+ expect(await methods.claimAgentBackgroundToolResultBatch(input('one'))).toEqual(current);+ });++ it('clears a collecting-only plan retired by a manual poll after its collector crashes', async () => {+ await ready('one');+ await ready('two');+ const write = Delivery.updateOne.bind(Delivery);+ const crash = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {+ const query = write(...args);+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResult.resultClaim'))+ throw new Error('crash before root claim');+ return query;+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(+ 'crash before root claim',+ );+ crash.mockRestore();+ const collecting = await Delivery.findOne({ deliveryKey: 'one' })+ .select('+backgroundToolResultBatch +backgroundToolResult')+ .lean();+ expect(collecting?.backgroundToolResultBatch).toMatchObject({ candidates: ['one', 'two'] });+ expect(collecting?.backgroundToolResult?.resultClaim).toBeUndefined();+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ settledAt: START,+ reason: 'manual poll',+ onlyIfUnclaimed: true,+ requireTransition: true,+ }),+ ).toBe(true);+ const retired = await Delivery.findOne({ deliveryKey: 'one' })+ .select('+backgroundToolResultBatch')+ .lean();+ expect(retired).toMatchObject({+ status: 'succeeded',+ expiresAt: new Date(START.getTime() + 90 * 24 * 60 * 60_000),+ });+ expect(retired).not.toHaveProperty('backgroundToolResultBatch');+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),+ ).toBeNull();+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(+ 'acquired',+ );+ });++ it('prevents a stale collector from acquiring siblings after empty-plan retirement', async () => {+ await ready('one');+ await ready('two');+ let pause: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const paused = new Promise<void>((resolve) => {+ pause = resolve;+ });+ const barrier = new Promise<void>((resolve) => {+ resume = resolve;+ });+ const write = Delivery.updateOne.bind(Delivery);+ const rootClaim = jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {+ const query = write(...args);+ if (JSON.stringify(query.getUpdate()).includes('backgroundToolResult.resultClaim')) {+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ pause();+ await barrier;+ return execute();+ });+ }+ return query;+ });+ const collecting = methods.claimAgentBackgroundToolResultBatch(input('one'));+ await paused;+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ settledAt: START,+ reason: 'manual poll',+ onlyIfUnclaimed: true,+ }),+ ).toBe(true);+ resume();+ expect(await collecting).toEqual({ status: 'not_ready' });+ rootClaim.mockRestore();+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),+ ).toBeNull();+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt');+ });++ it('retains an owned collecting plan before membership publication', async () => {+ await ready('one');+ await ready('two');+ const crash = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {+ throw new Error('crash before sibling claims');+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).rejects.toThrow(+ 'crash before sibling claims',+ );+ crash.mockRestore();+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ settledAt: START,+ reason: 'manual poll',+ onlyIfUnclaimed: true,+ }),+ ).toBe(false);+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ settledAt: START,+ reason: 'recovery',+ onlyIfDead: true,+ }),+ ).toBe(true);+ const retired = await Delivery.findOne({ deliveryKey: 'one' })+ .select('+backgroundToolResultBatch')+ .lean();+ expect(retired?.backgroundToolResultBatch).toBeDefined();+ expect(retired).not.toHaveProperty('expiresAt');+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ claimId: 'one',+ recoveryFenced: true,+ }),+ ).toBe(true);+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt');+ });++ it('retains a retired owner across the TTL horizon until every claim is released', async () => {+ await ready('one');+ await ready('two');+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (batch.status !== 'acquired') throw new Error('Expected batch');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ reason: 'recovery',+ onlyIfDead: true,+ settledAt: START,+ }),+ ).toBe(true);+ // The recovering process died before the generation fence or result release.+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt');+ const future = new Date(Date.now() + 91 * 24 * 60 * 60_000);+ await Delivery.deleteMany({ expiresAt: { $lte: future } });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'recoverable',+ });+ expect(+ await methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ claimId: 'one',+ batchId: batch.batchId,+ recoveryFenced: true,+ }),+ ).toBe(true);+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).toHaveProperty('expiresAt');+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(+ 'acquired',+ );+ });++ it('retains applied ownership until every receipt has durable proof of admission', async () => {+ await ready('one');+ await ready('two');+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (batch.status !== 'acquired') throw new Error('Expected batch');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ reason: 'recovery',+ onlyIfDead: true,+ settledAt: START,+ });+ const lostCopy = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {+ throw new Error('crash before receipt proof copy');+ });+ await expect(confirmBatch(input('one'))).rejects.toThrow('crash before receipt proof copy');+ lostCopy.mockRestore();+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt');+ await Delivery.deleteMany({+ expiresAt: { $lte: new Date(Date.now() + 91 * 24 * 60 * 60_000) },+ });+ expect(await Delivery.exists({ deliveryKey: 'one' })).not.toBeNull();+ // The durable root proof remains sufficient after the native key's horizon.+ await expect(methods.claimAgentBackgroundToolResultBatch(input('one'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'applied',+ });+ const root = await Delivery.findOne({ deliveryKey: 'one' })+ .select('+backgroundToolResultBatch')+ .lean();+ expect(root?.backgroundToolResultBatch?.proofCopiedAt).toBeInstanceOf(Date);+ expect(root?.expiresAt).toBeInstanceOf(Date);+ await Delivery.deleteOne({ deliveryKey: 'one' });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'applied',+ });+ });++ it.each(['before-copy', 'partial-copy', 'before-finish'] as const)(+ 'repairs a retired owner from its applied follower after %s failure',+ async (phase) => {+ await ready('one');+ await ready('two');+ await ready('three');+ const acquired = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (acquired.status !== 'acquired') throw new Error('Expected batch');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ reason: 'recovery',+ onlyIfDead: true,+ settledAt: START,+ });+ const copy = Delivery.updateMany.bind(Delivery);+ const finish = Delivery.updateOne.bind(Delivery);+ const failure =+ phase === 'before-finish'+ ? jest.spyOn(Delivery, 'updateOne').mockImplementation((...args) => {+ const query = finish(...args);+ if (JSON.stringify(query.getUpdate()).includes('proofCopiedAt'))+ throw new Error('proof completion failed');+ return query;+ })+ : jest.spyOn(Delivery, 'updateMany').mockImplementationOnce((...args) => {+ if (phase === 'before-copy') throw new Error('proof copy failed');+ const query = copy(+ { ...args[0], deliveryKey: { $in: ['one', 'two'] } },+ args[1],+ args[2],+ );+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ await execute();+ throw new Error('partial proof copy failed');+ });+ return query;+ });+ await expect(confirmBatch(input('one'))).rejects.toThrow();+ failure.mockRestore();+ expect(await Delivery.findOne({ deliveryKey: 'one' }).lean()).not.toHaveProperty('expiresAt');+ const cannotRepair = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {+ throw new Error('retry cannot copy');+ });+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).rejects.toThrow(+ 'retry cannot copy',+ );+ cannotRepair.mockRestore();+ await expect(+ methods.claimAgentBackgroundToolResultBatch(input('two')),+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' });+ const repaired = await Delivery.findOne({ deliveryKey: 'one' })+ .select('+backgroundToolResultBatch')+ .lean();+ expect(repaired?.backgroundToolResultBatch?.proofCopiedAt).toBeInstanceOf(Date);+ expect(repaired?.expiresAt).toBeInstanceOf(Date);+ for (const taskId of ['one', 'two', 'three']) {+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input(taskId), taskId }),+ ).toMatchObject({ appliedAt: expect.any(Date) });+ }+ await Delivery.deleteOne({ deliveryKey: 'one' });+ await expect(+ methods.claimAgentBackgroundToolResultBatch(input('three')),+ ).resolves.toMatchObject({ status: 'claimed', ownerStatus: 'applied' });+ },+ );++ it('does not publish a late collecting plan onto an already retired owner', async () => {+ await ready('one');+ await ready('two');+ let pause: () => void = () => undefined;+ let resume: () => void = () => undefined;+ const paused = new Promise<void>((resolve) => {+ pause = resolve;+ });+ const barrier = new Promise<void>((resolve) => {+ resume = resolve;+ });+ const update = Delivery.updateOne.bind(Delivery);+ const publication = jest.spyOn(Delivery, 'updateOne').mockImplementationOnce((...args) => {+ const query = update(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ pause();+ await barrier;+ return execute();+ });+ return query;+ });+ const stale = methods.claimAgentBackgroundToolResultBatch(input('one'));+ await paused;+ expect(+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ reason: 'discarded',+ onlyIfUnclaimed: true,+ settledAt: START,+ }),+ ).toBe(true);+ const collectingCrash = jest.spyOn(Delivery, 'updateMany').mockImplementationOnce(() => {+ throw new Error('crash after late publication');+ });+ resume();+ await expect(stale).resolves.toEqual({ status: 'not_ready' });+ publication.mockRestore();+ collectingCrash.mockRestore();+ expect(+ await Delivery.findOne({ deliveryKey: 'one' }).select('+backgroundToolResultBatch').lean(),+ ).not.toHaveProperty('backgroundToolResultBatch');+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),+ ).toBeNull();+ });++ it('cannot begin dispatch after the owner is retired, even without a queue token', async () => {+ await ready('one');+ await ready('two');+ const batch = await methods.claimAgentBackgroundToolResultBatch(input('one'));+ if (batch.status !== 'acquired') throw new Error('Expected batch');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityStatus: 'dead' } });+ await methods.retireAgentTriggerDelivery({+ deliveryKey: 'one',+ sourceId,+ reason: 'recovery',+ onlyIfDead: true,+ settledAt: START,+ });+ expect(+ await methods.beginAgentBackgroundToolResultBatchDispatch({+ ...input('one'),+ batchId: batch.batchId,+ dispatchId: 'late',+ }),+ ).toBe(false);+ });++ it('does not let an expired lease release its successor preparation', async () => {+ await ready('one');+ await ready('two');+ await Delivery.updateOne({ deliveryKey: 'one' }, { $set: { capabilityClaimToken: 'current' } });+ expect(+ (+ await methods.claimAgentBackgroundToolResultBatch({+ ...input('one'),+ deliveryClaimToken: 'current',+ })+ ).status,+ ).toBe('acquired');+ await expect(+ methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ claimId: 'one',+ deliveryClaimToken: 'expired',+ }),+ ).resolves.toBe(false);+ expect((await methods.claimAgentBackgroundToolResultBatch(input('two'))).status).toBe(+ 'claimed',+ );+ expect(await confirmBatch(input('one'))).toBe(true);+ await expect(+ methods.releaseAgentBackgroundToolResultClaims({+ ...input('one'),+ claimId: 'one',+ deliveryClaimToken: 'current',+ }),+ ).resolves.toBe(false);+ await expect(methods.claimAgentBackgroundToolResultBatch(input('two'))).resolves.toMatchObject({+ status: 'claimed',+ ownerStatus: 'applied',+ });+ });++ it('serializes definite release against an admission confirmation', async () => {+ await ready('one');+ await ready('two');+ await methods.claimAgentBackgroundToolResultBatch(input('one'));+ const [released, confirmed] = await Promise.all([+ methods.releaseAgentBackgroundToolResultClaims({ ...input('one'), claimId: 'one' }),+ confirmBatch(input('one')),+ ]);+ expect(Number(released) + Number(confirmed)).toBe(1);+ });++ it('uses bounded selection, including a batch size of one', async () => {+ await ready('one');+ await ready('two');+ expect(+ await methods.claimAgentBackgroundToolResultBatch({ ...input('one'), limit: 1 }),+ ).toMatchObject({ status: 'acquired', results: [{ taskId: 'one' }] });+ expect(+ await methods.getAgentBackgroundToolResultClaim({ ...input('two'), taskId: 'two' }),+ ).toBeNull();+ });++ it('keeps private batch state out of delivery reads and erases it with conversation results', async () => {+ await ready('one');+ await ready('two');+ await methods.claimAgentBackgroundToolResultBatch(input('one'));+ expect(await methods.getAgentTriggerDelivery('one')).not.toHaveProperty(+ 'backgroundToolResultBatch',+ );+ await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation']);+ const row = await Delivery.findOne({ deliveryKey: 'one' })+ .select('+backgroundToolResultBatch +backgroundToolResult')+ .lean();+ expect(row).not.toHaveProperty('backgroundToolResultBatch');+ expect(row).not.toHaveProperty('backgroundToolResult');+ });+});++describe('background receipt batch contention', () => {+ async function seed(+ groups = 1,+ capability = AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+ ) {+ const user = new mongoose.Types.ObjectId();+ const claims: Parameters<typeof methods.claimAgentBackgroundToolResultBatch>[0][] = [];+ for (let group = 0; group < groups; group++) {+ for (let task = 0; task < 4; task++) {+ const key = `group-${group}-task-${task}`;+ await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ deliveryKey: key,+ orderingKey: key,+ user,+ requiredWorkerCapability: capability,+ envelope: {+ event: {+ type: 'background-tool.completion',+ source: { id: 'background-tool-completion', type: 'internal' },+ payload: { taskId: key, toolCallId: key, toolName: 'tool' },+ },+ target: {+ conversationId: `conversation-${group}`,+ parentMessageId: 'parent',+ agentId: 'agent',+ },+ },+ }),+ );+ await methods.persistAgentBackgroundToolResult({+ deliveryKey: key,+ sourceId: 'background-tool-completion',+ result: { status: 'completed', output: key, settledAt: START },+ });+ claims.push({+ deliveryKey: key,+ sourceId: 'background-tool-completion',+ userId: user.toString(),+ tenantId: 'tenant-1',+ conversationId: `conversation-${group}`,+ parentMessageId: 'parent',+ agentId: 'agent',+ limit: 8,+ maxMetadataChars: 16000,+ });+ }+ }+ return claims;+ }++ it('makes progress when two workers cross the collecting barrier together', async () => {+ const claims = await seed();+ let arrivals = 0;+ let open: () => void = () => undefined;+ const barrier = new Promise<void>((resolve) => {+ open = resolve;+ });+ const updateMany = Delivery.updateMany.bind(Delivery);+ const race = jest.spyOn(Delivery, 'updateMany').mockImplementation((...args) => {+ const query = updateMany(...args);+ const execute = query.exec.bind(query);+ jest.spyOn(query, 'exec').mockImplementationOnce(async () => {+ if (++arrivals === 2) open();+ await barrier;+ return execute();+ });+ return query;+ });+ const batches = await Promise.all(+ claims.slice(0, 2).map((claim) => methods.claimAgentBackgroundToolResultBatch(claim)),+ );+ race.mockRestore();+ const consumed: string[] = [];+ for (let index = 0; index < batches.length; index++) {+ const batch = batches[index]!;+ if (batch.status !== 'acquired') continue;+ consumed.push(...batch.results.map((result) => result.taskId));+ await confirmBatch(claims[index]!);+ }+ for (const claim of claims) {+ const batch = await methods.claimAgentBackgroundToolResultBatch(claim);+ if (batch.status === 'acquired' && !consumed.includes(claim.deliveryKey)) {+ consumed.push(...batch.results.map((result) => result.taskId));+ await confirmBatch(claim);+ }+ }+ expect(new Set(consumed).size).toBe(4);+ expect(consumed).toHaveLength(4);+ });++ it('reduces 8 conversations by 4 ready receipts without duplicates or stranded results', async () => {+ const baselineClaims = await seed(+ 8,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ );+ let baselineTurns = 0;+ for (const claim of baselineClaims) {+ const batch = await methods.claimAgentBackgroundToolResults({+ ...claim,+ claimId: claim.deliveryKey,+ });+ if (batch.status === 'acquired') baselineTurns++;+ }+ await Promise.all([Delivery.deleteMany({}), LaneSequence.deleteMany({})]);+ const claims = await seed(8);+ const started = Date.now();+ const consumed: string[] = [];+ const latency: number[] = [];+ let turns = 0;+ // One ready root per conversation, as with a bounded worker pool behind+ // a busy parent. A separate barrier test covers same-conversation overlap.+ for (let round = 0; round < 4; round++) {+ await Promise.all(+ Array.from({ length: 8 }, async (_, group) => {+ const claim = claims[group * 4 + round]!;+ const batch = await methods.claimAgentBackgroundToolResultBatch(claim);+ if (batch.status !== 'acquired') return;+ turns++;+ consumed.push(...batch.results.map((result) => result.taskId));+ await confirmBatch(claim);+ latency.push(Date.now() - started);+ }),+ );+ }+ const unique = new Set(consumed).size;+ const report = {+ uniqueResults: unique,+ duplicates: consumed.length - unique,+ strandedResults: 32 - unique,+ baselineTurns,+ turns,+ admissionLatencyMs: Math.max(...latency),+ };+ console.info('BACKGROUND_RECEIPT_STRESS', JSON.stringify(report));+ expect(report).toMatchObject({+ uniqueResults: 32,+ duplicates: 0,+ strandedResults: 0,+ baselineTurns: 32,+ turns: 8,+ });+ });+});diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.tsindex 81d4d7140c..22cab0aded 100644--- a/packages/data-schemas/src/methods/triggerDelivery.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.ts@@ -1,4 +1,5 @@-import { createHash } from 'node:crypto';+import { createHash, randomUUID } from 'node:crypto';+import { backgroundResultMetadata } from 'librechat-data-provider';import type { FilterQuery, Model, Types } from 'mongoose';import type {AgentBackgroundToolResultReceipt,@@ -17,8 +18,10 @@ import type {IAgentTriggerUserPurge,IAgentTriggerUserPurgeDocument,} from '~/types/triggerDelivery';+import type { MessageMethods } from './message';import {AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,@@ -38,6 +41,10 @@ const DEFAULT_PURGE_RECOVERY_LIMIT = 25;const MAX_PURGE_RECOVERY_LIMIT = 200;const HISTORY_LIMIT = 64;const MAX_BATCH_SIZE = 8;+const BACKGROUND_RECEIPT_CAPABILITIES = [+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+];const MAX_BATCH_BYTES = 512 * 1024;/** Bounds the claim's compare-and-swap retries. Each lost round means another* worker claimed the row this one read, so the queue is making progress. */@@ -167,6 +174,48 @@ export type AgentBackgroundToolResultClaim =}>;};+export interface AgentBackgroundToolResultBatchInput {+ deliveryKey: string;+ sourceId: string;+ userId: string;+ tenantId?: string;+ conversationId: string;+ parentMessageId: string;+ agentId: string;+ limit: number;+ maxMetadataChars: number;+ deliveryClaimToken?: string;+}++export type AgentBackgroundToolResultBatchClaim =+ | { status: 'legacy' }+ | { status: 'not_ready'; waitingForResult?: true }+ | {+ status: 'claimed';+ claimId: string;+ batchId: string;+ ownerStatus: 'pending' | 'applied' | 'recoverable';+ }+ | (Extract<AgentBackgroundToolResultClaim, { status: 'acquired' }> & { batchId: string });++export type AgentBackgroundToolResultBatchOwnerInput = Omit<+ AgentBackgroundToolResultBatchInput,+ 'limit' | 'maxMetadataChars' | 'agentId'+> & { agentId?: string };++export type AgentBackgroundToolResultBatchReleaseInput = {+ sourceId: string;+ userId: string;+ conversationId: string;+ parentMessageId: string;+ claimId: string;+ deliveryClaimToken?: string;+ batchId?: string;+ dispatchId?: string;+ /** Retirement and generation-claim CAS proved that no admission can occur. */+ recoveryFenced?: true;+};+export interface AgentTriggerDeliveryFence {id: string;workerId: string;@@ -414,13 +463,21 @@ export interface AgentTriggerDeliveryMethods {claimId: string;limit?: number;}) => Promise<AgentBackgroundToolResultClaim>;- releaseAgentBackgroundToolResultClaims: (input: {- sourceId: string;- userId: string;- conversationId: string;- parentMessageId: string;- claimId: string;- }) => Promise<boolean>;+ releaseAgentBackgroundToolResultClaims: (+ input: AgentBackgroundToolResultBatchReleaseInput,+ ) => Promise<boolean>;+ claimAgentBackgroundToolResultBatch: (+ input: AgentBackgroundToolResultBatchInput,+ ) => Promise<AgentBackgroundToolResultBatchClaim>;+ getAgentBackgroundToolResultBatch: (+ input: AgentBackgroundToolResultBatchOwnerInput,+ ) => Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null>;+ beginAgentBackgroundToolResultBatchDispatch: (+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },+ ) => Promise<boolean>;+ confirmAgentBackgroundToolResultBatch: (+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },+ ) => Promise<boolean>;settleAgentTriggerHandlingOutcome: (input: SettleAgentTriggerHandlingOutcomeInput,) => Promise<boolean>;@@ -628,6 +685,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDelivexport function createAgentTriggerDeliveryMethods(mongoose: typeof import('mongoose'),deps: {+ releaseBatchProjections?: MessageMethods['releaseBackgroundToolResultClaims'];purgeQueuedTurnsForUser?: (user: string | Types.ObjectId) => Promise<unknown>;} = {},): AgentTriggerDeliveryMethods {@@ -1125,6 +1183,8 @@ export function createAgentTriggerDeliveryMethods(input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&input.requiredWorkerCapability !==AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&+ input.requiredWorkerCapability !==+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 &&input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 &&input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2@@ -2182,38 +2242,70 @@ export function createAgentTriggerDeliveryMethods(},};})();- const retired = await Delivery()- .findOneAndUpdate(- {- deliveryKey: input.deliveryKey,- 'envelope.event.source.type': 'internal',- 'envelope.event.source.id': input.sourceId,- ...statusFence,- },- {- $set: {- status: 'succeeded',- result,- settledAt: input.settledAt,- expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),- laneCleanupPendingAt: input.settledAt,+ const retire = (retainBatch: boolean, discardEmptyPlan = false) =>+ Delivery()+ .findOneAndUpdate(+ {+ deliveryKey: input.deliveryKey,+ 'envelope.event.source.type': 'internal',+ 'envelope.event.source.id': input.sourceId,+ ...statusFence,+ ...(discardEmptyPlan && {+ backgroundToolResultBatch: { $exists: true },+ 'backgroundToolResultBatch.members.0': { $exists: false },+ 'backgroundToolResult.resultClaim.claimId': { $ne: input.deliveryKey },+ }),+ ...(!discardEmptyPlan &&+ retainBatch && {+ backgroundToolResultBatch: { $exists: true },+ 'backgroundToolResultBatch.proofCopiedAt': { $exists: false },+ }),+ ...(!discardEmptyPlan &&+ !retainBatch && {+ $and: [+ {+ $or: [+ { backgroundToolResultBatch: { $exists: false } },+ { 'backgroundToolResultBatch.proofCopiedAt': { $exists: true } },+ ],+ },+ ],+ }),+ ...(input.onlyIfUnclaimed === true+ ? { 'backgroundToolResult.resultClaim': { $exists: false } }+ : {}),},- $unset: {- leaseBy: 1,- leaseUntil: 1,- claimToken: 1,- capabilityStatus: 1,- claimAvailableAt: 1,- capabilityLeaseBy: 1,- capabilityLeaseUntil: 1,- capabilityClaimToken: 1,- lastError: 1,+ {+ $set: {+ status: 'succeeded',+ result,+ settledAt: input.settledAt,+ ...(!retainBatch && {+ expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),+ }),+ laneCleanupPendingAt: input.settledAt,+ },+ $unset: {+ leaseBy: 1,+ leaseUntil: 1,+ claimToken: 1,+ capabilityStatus: 1,+ claimAvailableAt: 1,+ capabilityLeaseBy: 1,+ capabilityLeaseUntil: 1,+ capabilityClaimToken: 1,+ lastError: 1,+ ...(retainBatch && { expiresAt: 1 }),+ ...(discardEmptyPlan && { backgroundToolResultBatch: 1 }),+ },},- },- { new: true },- )- .select('_id orderingKey laneCleanupPendingAt')- .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();+ { new: true },+ )+ .select('_id orderingKey laneCleanupPendingAt')+ .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();+ // A retired owner is its followers' recovery and frozen-membership record.+ // Its TTL cannot start while release or receipt-proof copying is incomplete.+ const retired = (await retire(false)) ?? (await retire(false, true)) ?? (await retire(true));if (retired?._id != null) {try {await fulfillLaneCleanupRequest(retired);@@ -2268,7 +2360,7 @@ export function createAgentTriggerDeliveryMethods(requiredWorkerCapability: {$in: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,- AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ ...BACKGROUND_RECEIPT_CAPABILITIES,],},'envelope.event.source.type': 'internal',@@ -2312,7 +2404,7 @@ export function createAgentTriggerDeliveryMethods(requiredWorkerCapability: {$in: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,- AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ ...BACKGROUND_RECEIPT_CAPABILITIES,],},'envelope.event.source.type': 'internal',@@ -2366,7 +2458,7 @@ export function createAgentTriggerDeliveryMethods('envelope.target.conversationId': input.conversationId,...(input.taskId != null && { 'envelope.event.payload.taskId': input.taskId }),/** Legacy rows keep results only on the parent message. */- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },};const read = (filter: FilterQuery<IAgentTriggerDelivery>) =>Delivery()@@ -2568,7 +2660,7 @@ export function createAgentTriggerDeliveryMethods(}const identity = {deliveryKey: input.deliveryKey,- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },'envelope.event.source.type': 'internal','envelope.event.source.id': input.sourceId,backgroundToolResultErasedAt: { $exists: false },@@ -2612,7 +2704,7 @@ export function createAgentTriggerDeliveryMethods(const delivery = await Delivery().findOne({deliveryKey: input.deliveryKey,- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },'envelope.event.source.type': 'internal','envelope.event.source.id': input.sourceId,})@@ -2648,7 +2740,7 @@ export function createAgentTriggerDeliveryMethods(}) {return {user: input.userId,- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },'envelope.event.source.type': 'internal','envelope.event.source.id': input.sourceId,'envelope.target.conversationId': input.conversationId,@@ -2769,38 +2861,592 @@ export function createAgentTriggerDeliveryMethods(return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };}- async function releaseAgentBackgroundToolResultClaims(input: {- sourceId: string;- userId: string;- conversationId: string;- parentMessageId: string;- claimId: string;- }): Promise<boolean> {+ async function releaseAgentBackgroundToolResultClaims(+ input: AgentBackgroundToolResultBatchReleaseInput,+ ): Promise<boolean> {if (- input.sourceId.length === 0 ||- input.sourceId.length > 256 ||- input.userId.length === 0 ||- input.conversationId.length === 0 ||- input.conversationId.length > 256 ||- input.parentMessageId.length === 0 ||- input.parentMessageId.length > 256 ||- input.claimId.length === 0 ||+ [+ input.sourceId,+ input.userId,+ input.conversationId,+ input.parentMessageId,+ input.claimId,+ ].some((value) => value.length === 0 || value.length > 256) ||input.claimId.length > 128) {throw new TypeError('Invalid background tool result receipt claim release');}const scope = backgroundResultIdentity(input);+ const releaseId = randomUUID();+ const releasing = await Delivery()+ .findOneAndUpdate(+ {+ ...scope,+ deliveryKey: input.claimId,+ backgroundToolResultBatch: { $exists: true },+ 'backgroundToolResultBatch.appliedAt': { $exists: false },+ ...(input.batchId == null ? {} : { 'backgroundToolResultBatch.batchId': input.batchId }),+ ...(input.deliveryClaimToken == null+ ? {}+ : { capabilityClaimToken: input.deliveryClaimToken }),+ ...(input.recoveryFenced === true+ ? {}+ : {+ $or: [+ { 'backgroundToolResultBatch.releasing': true },+ { 'backgroundToolResultBatch.dispatchCount': 0 },+ ...(input.dispatchId == null+ ? []+ : [+ {+ 'backgroundToolResultBatch.dispatchCount': 1,+ 'backgroundToolResultBatch.dispatchId': input.dispatchId,+ },+ ]),+ ],+ }),+ },+ {+ $set: {+ 'backgroundToolResultBatch.releasing': true,+ 'backgroundToolResultBatch.releaseId': releaseId,+ },+ },+ { new: true, timestamps: false },+ )+ .select('status +backgroundToolResultBatch')+ .lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();+ if (+ releasing == null &&+ (await Delivery().exists({+ ...scope,+ deliveryKey: input.claimId,+ backgroundToolResultBatch: { $exists: true },+ })) != null+ )+ return false;+ const batch = releasing?.backgroundToolResultBatch;+ const batchId = batch?.batchId ?? input.batchId;+ const receiptScope = {+ ...scope,+ 'backgroundToolResult.resultClaim.claimId': input.claimId,+ 'backgroundToolResult.resultClaim.batchId': batchId ?? null,+ };await Delivery().updateMany(- { ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },+ { ...receiptScope, 'backgroundToolResult.resultClaim.appliedAt': { $exists: false } },{ $unset: { 'backgroundToolResult.resultClaim': 1 } },{ timestamps: false },);- return (+ if ((await Delivery().exists(receiptScope)) != null) return false;+ if (batch != null) {+ if (deps.releaseBatchProjections == null)+ throw new Error('Background batch projection cleanup is unavailable');+ if (+ !(await deps.releaseBatchProjections({+ userId: input.userId,+ conversationId: input.conversationId,+ messageId: input.parentMessageId,+ kind: 'wakeup',+ claimId: input.claimId,+ batchId: batch.batchId,+ allowMissingMessage: true,+ }))+ )+ return false;+ const clear = (retired: boolean) =>+ Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.claimId,+ 'backgroundToolResultBatch.batchId': batch.batchId,+ 'backgroundToolResultBatch.releaseId': releaseId,+ 'backgroundToolResultBatch.appliedAt': { $exists: false },+ status: retired ? 'succeeded' : { $ne: 'succeeded' },+ ...(input.deliveryClaimToken == null+ ? {}+ : { capabilityClaimToken: input.deliveryClaimToken }),+ },+ {+ $unset: { backgroundToolResultBatch: 1 },+ ...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),+ },+ { timestamps: false },+ );+ const cleared = await clear(releasing?.status === 'succeeded');+ if (cleared.matchedCount === 1) return true;+ return releasing?.status !== 'succeeded' && (await clear(true)).matchedCount === 1;+ }+ return true;+ }++ function backgroundBatchScope(+ input:+ | AgentBackgroundToolResultBatchInput+ | Omit<AgentBackgroundToolResultBatchInput, 'limit' | 'maxMetadataChars'>,+ ) {+ return {+ ...backgroundResultIdentity(input),+ tenantId: input.tenantId ?? null,+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+ 'envelope.event.type': 'background-tool.completion',+ };+ }++ /** Each result has its own CAS. Candidates survive a collecting crash; only+ * confirmed owners enter the frozen member list used by every retry. */+ async function claimAgentBackgroundToolResultBatch(+ input: AgentBackgroundToolResultBatchInput,+ ): Promise<AgentBackgroundToolResultBatchClaim> {+ if (+ [+ input.deliveryKey,+ input.sourceId,+ input.userId,+ input.conversationId,+ input.parentMessageId,+ input.agentId,+ ].some((value) => value.length === 0 || value.length > 256) ||+ input.deliveryKey.length > 128+ ) {+ throw new TypeError('Invalid background result batch identity');+ }+ if (+ !Number.isSafeInteger(input.limit) ||+ input.limit < 1 ||+ !Number.isSafeInteger(input.maxMetadataChars) ||+ input.maxMetadataChars < 1+ ) {+ throw new TypeError('Invalid background result batch bounds');+ }+ const scope = backgroundBatchScope(input);+ const projection =+ 'deliveryKey requiredWorkerCapability status capabilityStatus result +backgroundToolResult +backgroundToolResultBatch';+ type Row = Pick<+ IAgentTriggerDelivery,+ | 'deliveryKey'+ | 'requiredWorkerCapability'+ | 'status'+ | 'capabilityStatus'+ | 'result'+ | 'backgroundToolResult'+ | 'backgroundToolResultBatch'+ > & { envelope?: unknown };+ const readRoot = () =>+ Delivery()+ .findOne({ ...scope, deliveryKey: input.deliveryKey })+ .select(projection + ' envelope')+ .lean<Row>();+ let root = await readRoot();+ if (root == null) {+ const gated = await Delivery().exists({+ user: input.userId,+ deliveryKey: input.deliveryKey,+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3,+ });+ return { status: gated == null ? 'legacy' : 'not_ready' };+ }+ if (root.backgroundToolResultBatch?.appliedAt != null) {+ if (+ root.backgroundToolResultBatch.proofCopiedAt == null &&+ !(await confirmAgentBackgroundToolResultBatch({+ ...input,+ batchId: root.backgroundToolResultBatch.batchId,+ }))+ )+ return { status: 'not_ready' };+ return {+ status: 'claimed',+ claimId: input.deliveryKey,+ batchId: root.backgroundToolResultBatch.batchId,+ ownerStatus: 'applied',+ };+ }+ if (root.backgroundToolResultBatch?.releasing === true) {+ const released = await releaseAgentBackgroundToolResultClaims({+ ...input,+ claimId: input.deliveryKey,+ batchId: root.backgroundToolResultBatch.batchId,+ });+ return released ? claimAgentBackgroundToolResultBatch(input) : { status: 'not_ready' };+ }+ if (root.backgroundToolResult == null) return { status: 'not_ready', waitingForResult: true };+ const claim = root.backgroundToolResult.resultClaim;+ if (+ claim?.claimId === input.deliveryKey &&+ claim.batchId !== root.backgroundToolResultBatch?.batchId+ ) {+ await Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,+ 'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,+ 'backgroundToolResult.resultClaim.appliedAt': { $exists: false },+ },+ { $unset: { 'backgroundToolResult.resultClaim': 1 } },+ { timestamps: false },+ );+ return { status: 'not_ready' };+ }+ if (claim != null && claim.claimId !== input.deliveryKey) {+ if (+ root.backgroundToolResultBatch != null &&+ root.backgroundToolResultBatch.appliedAt == null+ ) {+ await releaseAgentBackgroundToolResultClaims({+ ...input,+ claimId: input.deliveryKey,+ batchId: root.backgroundToolResultBatch.batchId,+ });+ }+ const owner = await Delivery()+ .findOne({ ...scope, deliveryKey: claim.claimId })+ .select(projection)+ .lean<Row>();+ const batch = owner?.backgroundToolResultBatch;+ const member = batch?.members?.includes(input.deliveryKey);+ if (member && batch?.batchId === claim.batchId && batch?.appliedAt != null) {+ // Retirement removed this owner from the queue. Its followers must+ // finish the proof copy before they can all settle and abandon it.+ if (+ batch.proofCopiedAt == null &&+ !(await confirmAgentBackgroundToolResultBatch({+ ...input,+ deliveryKey: claim.claimId,+ batchId: batch.batchId,+ }))+ )+ return { status: 'not_ready' };+ return {+ status: 'claimed',+ claimId: claim.claimId,+ batchId: claim.batchId!,+ ownerStatus: 'applied',+ };+ }+ if (claim.appliedAt != null)+ return {+ status: 'claimed',+ claimId: claim.claimId,+ batchId: claim.batchId!,+ ownerStatus: 'applied',+ };+ // A stale collector may finish a CAS after another collector froze a subset.+ if (+ (batch?.members != null && !member) ||+ (batch != null && batch.batchId !== claim.batchId)+ ) {+ await Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ 'backgroundToolResult.resultClaim.claimId': claim.claimId,+ 'backgroundToolResult.resultClaim.batchId': claim.batchId ?? null,+ 'backgroundToolResult.resultClaim.appliedAt': { $exists: false },+ },+ { $unset: { 'backgroundToolResult.resultClaim': 1 } },+ { timestamps: false },+ );+ return { status: 'not_ready' };+ }+ const retired =+ owner?.result != null &&+ typeof owner.result === 'object' &&+ 'backgroundToolCompletionRetired' in owner.result;+ const recoverable =+ owner == null ||+ DEAD_STATUSES.includes(owner.status) ||+ owner.capabilityStatus === 'dead' ||+ retired;+ return {+ status: 'claimed',+ claimId: claim.claimId,+ batchId: claim.batchId!,+ ownerStatus: recoverable ? 'recoverable' : 'pending',+ };+ }+ if (root.backgroundToolResultBatch == null) {+ const candidates =+ input.limit === 1+ ? []+ : await Delivery()+ .find({+ ...scope,+ deliveryKey: { $ne: input.deliveryKey },+ status: { $in: UNDELIVERED_STATUSES },+ capabilityStatus: { $ne: 'dead' },+ backgroundToolResult: { $exists: true },+ 'backgroundToolResult.resultClaim': { $exists: false },+ })+ .select('deliveryKey envelope +backgroundToolResult')+ .sort({ createdAt: 1, _id: 1 })+ .limit(Math.min(input.limit, 32) - 1)+ .lean<Row[]>();+ const requested = projectBackgroundResult(root);+ if (requested == null) return { status: 'not_ready' };+ const metadataCost = (result: NonNullable<ReturnType<typeof projectBackgroundResult>>) =>+ JSON.stringify(backgroundResultMetadata(result)).length + 1;+ let remaining = input.maxMetadataChars - metadataCost(requested) - 1;+ if (remaining < 0) throw new TypeError('Background result metadata exceeds its input budget');+ const batchId = randomUUID();+ const keys = [input.deliveryKey];+ for (const candidate of candidates) {+ const result = projectBackgroundResult(candidate);+ if (result == null) continue;+ const cost = metadataCost(result);+ if (cost > remaining) continue;+ remaining -= cost;+ keys.push(candidate.deliveryKey);+ }+ const planned = await Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ backgroundToolResultBatch: { $exists: false },+ status: { $in: UNDELIVERED_STATUSES },+ capabilityStatus: { $ne: 'dead' },+ ...(input.deliveryClaimToken == null+ ? {}+ : { capabilityClaimToken: input.deliveryClaimToken }),+ $or: [+ { 'backgroundToolResult.resultClaim': { $exists: false } },+ { 'backgroundToolResult.resultClaim.claimId': input.deliveryKey },+ ],+ },+ { $set: { backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } } },+ { timestamps: false },+ );+ root =+ planned.modifiedCount === 1+ ? { ...root, backgroundToolResultBatch: { batchId, dispatchCount: 0, candidates: keys } }+ : await readRoot();+ if (root == null) return { status: 'not_ready' };+ }+ let batch = root.backgroundToolResultBatch;+ if (batch == null) return { status: 'not_ready' };+ if (batch.members == null) {+ // Retirement of a collecting-only plan contends with this root CAS.+ // No sibling can depend on the owner until its own receipt is claimed.+ if (root.backgroundToolResult?.resultClaim?.claimId !== input.deliveryKey) {+ const claimedRoot = await Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ status: { $in: UNDELIVERED_STATUSES },+ capabilityStatus: { $ne: 'dead' },+ 'backgroundToolResultBatch.batchId': batch.batchId,+ 'backgroundToolResultBatch.releasing': { $ne: true },+ 'backgroundToolResult.resultClaim': { $exists: false },+ ...(input.deliveryClaimToken == null+ ? {}+ : { capabilityClaimToken: input.deliveryClaimToken }),+ },+ {+ $set: {+ 'backgroundToolResult.resultClaim': {+ kind: 'wakeup',+ claimId: input.deliveryKey,+ claimedAt: new Date(),+ batchId: batch.batchId,+ },+ },+ },+ { timestamps: false },+ );+ if (claimedRoot.matchedCount !== 1) return { status: 'not_ready' };+ }+ await Delivery().updateMany(+ {+ ...scope,+ deliveryKey: { $in: batch.candidates },+ status: { $in: UNDELIVERED_STATUSES },+ capabilityStatus: { $ne: 'dead' },+ backgroundToolResult: { $exists: true },+ 'backgroundToolResult.resultClaim': { $exists: false },+ },+ {+ $set: {+ 'backgroundToolResult.resultClaim': {+ kind: 'wakeup',+ claimId: input.deliveryKey,+ claimedAt: new Date(),+ batchId: batch.batchId,+ },+ },+ },+ { timestamps: false },+ );+ const owned = await Delivery()+ .find({+ ...scope,+ deliveryKey: { $in: batch.candidates },+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,+ 'backgroundToolResult.resultClaim.batchId': batch.batchId,+ })+ .select('deliveryKey')+ .lean<Pick<IAgentTriggerDelivery, 'deliveryKey'>[]>();+ const owners = new Set(owned.map((row) => row.deliveryKey));+ const members = batch.candidates.filter((key) => owners.has(key));+ if (!owners.has(input.deliveryKey)) {+ await releaseAgentBackgroundToolResultClaims({+ ...input,+ claimId: input.deliveryKey,+ batchId: batch.batchId,+ });+ return { status: 'not_ready' };+ }+ const frozen = await Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,+ 'backgroundToolResultBatch.batchId': batch.batchId,+ 'backgroundToolResultBatch.releasing': { $ne: true },+ ...(input.deliveryClaimToken == null+ ? {}+ : { capabilityClaimToken: input.deliveryClaimToken }),+ 'backgroundToolResultBatch.members': { $exists: false },+ },+ { $set: { 'backgroundToolResultBatch.members': members } },+ { timestamps: false },+ );+ if (frozen.modifiedCount === 1) {+ batch = { ...batch, members };+ } else {+ root = await readRoot();+ batch = root?.backgroundToolResultBatch;+ }+ }+ if (batch?.members == null || !batch.members.includes(input.deliveryKey))+ return { status: 'not_ready' };+ const rows = await Delivery()+ .find({+ ...scope,+ deliveryKey: { $in: batch.members },+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,+ 'backgroundToolResult.resultClaim.batchId': batch.batchId,+ })+ .select('deliveryKey envelope +backgroundToolResult')+ .lean<Row[]>();+ const byKey = new Map(rows.map((row) => [row.deliveryKey, projectBackgroundResult(row)]));+ const results = batch.members.map((key) => byKey.get(key));+ if (results.some((result) => result == null)) return { status: 'not_ready' };+ if (+ input.deliveryClaimToken != null &&(await Delivery().exists({...scope,- 'backgroundToolResult.resultClaim.claimId': input.claimId,+ deliveryKey: input.deliveryKey,+ capabilityClaimToken: input.deliveryClaimToken,+ 'backgroundToolResultBatch.batchId': batch.batchId,+ 'backgroundToolResultBatch.releasing': { $ne: true },})) == null+ ) {+ return { status: 'not_ready' };+ }+ return {+ status: 'acquired',+ batchId: batch.batchId,+ results: results.filter((result): result is NonNullable<typeof result> => result != null),+ };+ }++ async function getAgentBackgroundToolResultBatch(+ input: AgentBackgroundToolResultBatchOwnerInput,+ ): Promise<IAgentTriggerDelivery['backgroundToolResultBatch'] | null> {+ const row = await Delivery()+ .findOne({ ...backgroundResultIdentity(input), deliveryKey: input.deliveryKey })+ .select('+backgroundToolResultBatch')+ .lean<Pick<IAgentTriggerDelivery, 'backgroundToolResultBatch'>>();+ return row?.backgroundToolResultBatch ?? null;+ }++ async function beginAgentBackgroundToolResultBatchDispatch(+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string; dispatchId: string },+ ): Promise<boolean> {+ const marked = await Delivery().updateOne(+ {+ ...backgroundResultIdentity(input),+ deliveryKey: input.deliveryKey,+ status: { $in: UNDELIVERED_STATUSES },+ capabilityStatus: { $ne: 'dead' },+ 'backgroundToolResultBatch.batchId': input.batchId,+ 'backgroundToolResultBatch.members.0': { $exists: true },+ 'backgroundToolResultBatch.releasing': { $ne: true },+ 'backgroundToolResultBatch.appliedAt': { $exists: false },+ ...(input.deliveryClaimToken == null+ ? {}+ : { capabilityClaimToken: input.deliveryClaimToken }),+ },+ {+ $inc: { 'backgroundToolResultBatch.dispatchCount': 1 },+ $set: { 'backgroundToolResultBatch.dispatchId': input.dispatchId },+ },+ { timestamps: false },+ );+ return marked.matchedCount === 1;+ }++ /** Admission proof precedes follower settlement, including lost HTTP replies. */+ async function confirmAgentBackgroundToolResultBatch(+ input: AgentBackgroundToolResultBatchOwnerInput & { batchId: string },+ ): Promise<boolean> {+ const scope = backgroundResultIdentity(input);+ const root = await Delivery()+ .findOneAndUpdate(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ 'backgroundToolResultBatch.members.0': { $exists: true },+ 'backgroundToolResultBatch.batchId': input.batchId,+ 'backgroundToolResultBatch.releasing': { $ne: true },+ },+ { $min: { 'backgroundToolResultBatch.appliedAt': new Date() } },+ { new: true, timestamps: false },+ )+ .select('status +backgroundToolResultBatch')+ .lean<Pick<IAgentTriggerDelivery, 'status' | 'backgroundToolResultBatch'>>();+ if (root?.backgroundToolResultBatch?.appliedAt == null) return false;+ await Delivery().updateMany(+ {+ ...scope,+ deliveryKey: { $in: root.backgroundToolResultBatch.members },+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,+ 'backgroundToolResult.resultClaim.batchId': input.batchId,+ },+ {+ $set: {+ 'backgroundToolResult.resultClaim.appliedAt': root.backgroundToolResultBatch.appliedAt,+ },+ },+ { timestamps: false },);+ if (+ (await Delivery().exists({+ ...scope,+ 'backgroundToolResult.resultClaim.claimId': input.deliveryKey,+ 'backgroundToolResult.resultClaim.batchId': input.batchId,+ 'backgroundToolResult.resultClaim.appliedAt': { $exists: false },+ })) != null+ )+ return false;+ const finish = (retired: boolean) =>+ Delivery().updateOne(+ {+ ...scope,+ deliveryKey: input.deliveryKey,+ 'backgroundToolResultBatch.batchId': input.batchId,+ 'backgroundToolResultBatch.appliedAt': { $exists: true },+ status: retired ? 'succeeded' : { $ne: 'succeeded' },+ },+ {+ $min: { 'backgroundToolResultBatch.proofCopiedAt': new Date() },+ ...(retired && { $max: { expiresAt: new Date(Date.now() + SUCCESS_RETENTION_MS) } }),+ },+ { timestamps: false },+ );+ const finished = await finish(root.status === 'succeeded');+ if (finished.matchedCount === 1) return true;+ return root.status !== 'succeeded' && (await finish(true)).matchedCount === 1;}async function settleAgentTriggerHandlingOutcome(@@ -3715,8 +4361,7 @@ export function createAgentTriggerDeliveryMethods(const error = normalizeFailure(input.error);const receiptFence = recoverReceipt? {- requiredWorkerCapability:- AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },backgroundToolResult: { $exists: true },}: {};@@ -4273,7 +4918,11 @@ export function createAgentTriggerDeliveryMethods(},{$set: { backgroundToolResultErasedAt: erasedAt },- $unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },+ $unset: {+ backgroundToolResult: 1,+ backgroundToolResultBatch: 1,+ backgroundToolResultDeletionPendingAt: 1,+ },},{ timestamps: false },);@@ -4287,7 +4936,7 @@ export function createAgentTriggerDeliveryMethods(await Delivery().updateMany({user,- requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ requiredWorkerCapability: { $in: BACKGROUND_RECEIPT_CAPABILITIES },'envelope.target.conversationId': { $in: conversationIds },backgroundToolResultErasedAt: { $exists: false },},@@ -4316,6 +4965,10 @@ export function createAgentTriggerDeliveryMethods(getAgentBackgroundToolResult,getAgentBackgroundToolResultClaim,claimAgentBackgroundToolResults,+ claimAgentBackgroundToolResultBatch,+ confirmAgentBackgroundToolResultBatch,+ beginAgentBackgroundToolResultBatchDispatch,+ getAgentBackgroundToolResultBatch,releaseAgentBackgroundToolResultClaims,settleAgentTriggerHandlingOutcome,admitAgentEventActorAction,diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.tsindex 927e421b3d..4db14e4d96 100644--- a/packages/data-schemas/src/schema/triggerDelivery.ts+++ b/packages/data-schemas/src/schema/triggerDelivery.ts@@ -168,6 +168,8 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(kind: { type: String, enum: ['wakeup'], required: true },claimId: { type: String, required: true, maxlength: 128 },claimedAt: { type: Date, required: true },+ appliedAt: { type: Date },+ batchId: { type: String, maxlength: 128 },},{ _id: false },),@@ -178,6 +180,23 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(),select: false,},+ backgroundToolResultBatch: {+ type: new Schema(+ {+ batchId: { type: String, required: true, maxlength: 128 },+ dispatchCount: { type: Number, required: true, min: 0 },+ dispatchId: { type: String, maxlength: 128 },+ releaseId: { type: String, maxlength: 128 },+ proofCopiedAt: { type: Date },+ candidates: { type: [String], required: true },+ releasing: { type: Boolean },+ members: { type: [String], default: undefined },+ appliedAt: { type: Date },+ },+ { _id: false },+ ),+ select: false,+ },backgroundToolResultErasedAt: { type: Date, select: false },backgroundToolResultDeletionPendingAt: { type: Date, select: false },attempts: { type: Number, required: true, default: 0, min: 0 },diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.tsindex 73ede79450..694dfc4991 100644--- a/packages/data-schemas/src/types/triggerDelivery.ts+++ b/packages/data-schemas/src/types/triggerDelivery.ts@@ -17,6 +17,8 @@ export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 ='background_tool_completion_v1';export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 ='background_tool_completion_receipt_v2';+export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_BATCH_V3 =+ 'background_tool_completion_batch_v3';export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V2 = 'agent_queued_turn_v2';export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;@@ -92,6 +94,8 @@ export interface AgentBackgroundToolResultReceipt {kind: 'wakeup';claimId: string;claimedAt: Date;+ appliedAt?: Date;+ batchId?: string;};}@@ -128,6 +132,18 @@ export interface IAgentTriggerDelivery {producerLeaseUntil?: Date;/** Durable source of truth for a background completion. */backgroundToolResult?: AgentBackgroundToolResultReceipt;+ /** Receipt batching is separate from event-envelope coalescing. */+ backgroundToolResultBatch?: {+ batchId: string;+ dispatchCount: number;+ dispatchId?: string;+ releaseId?: string;+ proofCopiedAt?: Date;+ candidates: string[];+ members?: string[];+ appliedAt?: Date;+ releasing?: true;+ };/** Conversation-deletion fence preventing a late producer from restoring private output. */backgroundToolResultErasedAt?: Date;backgroundToolResultDeletionPendingAt?: Date;