Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🐢 fix: Back Off Waiting Completion Wake-ups and Keep Their Trace Out of Requests (#16349)
Danny Avila3788fe05d92e
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.jsindex 7b58b54a9e..329bfd4635 100644--- a/api/server/services/Agents/triggers.js+++ b/api/server/services/Agents/triggers.js@@ -24,11 +24,13 @@ const getGenerationAdmissionEvidence = (userId, clientRequestId, streamId, conveconst subagentCompletionAdapter = createSubagentCompletionWakeupResolver({methods,getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),+ getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),});const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupResolver({methods,getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),+ getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),});const eventActorAdapter = createAgentEventContinueResolver({methods,diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.tsindex 1e87d6a2f5..ceb95fe540 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts@@ -756,6 +756,88 @@ describe('background tool completion wakeups', () => {expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();});+ it('backs off by waiting age while the invoking generation keeps running', async () => {+ const { methods } = resolverMethods();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => ({ status: 'running' }),+ });+ const deliveryEnvelope = await envelope();++ jest.setSystemTime(NOW + 120_000);+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'PARENT_NOT_READY', retryAfter: '12', deferWithoutAttempt: true },+ );+ jest.setSystemTime(NOW + 6 * 60 * 60_000);+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'PARENT_NOT_READY', retryAfter: '60' },+ );+ });++ it('caps the waiting backoff at the configured interval', async () => {+ const { methods } = resolverMethods();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => ({ status: 'running' }),+ getWaitMaxIntervalMs: () => 20_000,+ });+ const deliveryEnvelope = await envelope();++ jest.setSystemTime(NOW + 6 * 60 * 60_000);+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'PARENT_NOT_READY', retryAfter: '20' },+ );+ });++ it('keeps backing off while the parent is paused for approval', async () => {+ const { methods } = resolverMethods();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => ({ status: 'requires_action' }),+ });+ const deliveryEnvelope = await envelope();++ jest.setSystemTime(NOW + 60 * 60_000);+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'PARENT_NOT_READY', retryAfter: '60' },+ );+ });++ it('re-checks within a second once the parent has settled and only persistence remains', async () => {+ const { methods } = resolverMethods();+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => ({+ status: 'complete',+ metadata: { terminalPersistencePending: true },+ }),+ });+ const deliveryEnvelope = await envelope();++ jest.setSystemTime(NOW + 10 * 60_000);+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'PARENT_NOT_READY', retryAfter: '1' },+ );+ });++ it('backs off by waiting age while the tool result is not durable yet', async () => {+ const { methods } = resolverMethods();+ methods.claimBackgroundToolResults.mockResolvedValue({ status: 'missing', results: [] });+ const resolve = createBackgroundToolCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ });+ const deliveryEnvelope = await envelope();++ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'BACKGROUND_TOOL_RESULT_NOT_READY', retryAfter: '5' },+ );+ jest.setSystemTime(NOW + 5 * 60_000);+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(+ { code: 'BACKGROUND_TOOL_RESULT_NOT_READY', retryAfter: '30' },+ );+ });+it('does not manufacture terminal evidence from wall-clock age', async () => {const { methods } = resolverMethods();methods.claimBackgroundToolResults.mockResolvedValue({ status: 'missing', results: [] });diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.tsindex bfdb9f21d3..1ca6fca26c 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts@@ -21,6 +21,7 @@ import type {import type { AgentContinueTriggerEnvelope } from './triggers/envelope';import type { AgentTriggerDispatchContext } from './triggers/dispatch';import type { AgentTriggerEnqueueOptions } from './triggers/delivery';+import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';import { BACKGROUND_TOOL_PRODUCER_LEASE_MS } from './backgroundCompletion';import { createAgentTriggerEnvelope } from './triggers/envelope';import { AgentTriggerExecutionError } from './triggers/host';@@ -94,6 +95,8 @@ export interface BackgroundToolCompletionWakeupResolverDeps {methods: WakeupMethods;getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;getResultBatchSize?: () => number | undefined;+ /** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */+ getWaitMaxIntervalMs?: () => number | undefined;}function executionError(@@ -152,6 +155,12 @@ function isParentActive(job: GenerationState | null): boolean {);}+/** A running or approval-paused parent can stay busy for hours; one that has+ * settled and is only finishing terminal persistence clears within moments. */+function isParentWorking(job: GenerationState | null): boolean {+ return job?.status === 'running' || job?.status === 'requires_action';+}+function timestamp(message: Pick<IMessage, 'createdAt'>): number {const value = message.createdAt;if (value instanceof Date) {@@ -252,9 +261,12 @@ export function createBackgroundToolCompletionWakeupResolver({methods,getGenerationJob,getResultBatchSize,+ getWaitMaxIntervalMs,}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<AgentTriggerExecutionHostDeps['prepareContinue']> {+ const waitingRetry = (receivedAt: number): string =>+ waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);return async (envelope: AgentContinueTriggerEnvelope,context: AgentTriggerDispatchContext,@@ -288,7 +300,7 @@ export function createBackgroundToolCompletionWakeupResolver({code: 'PARENT_NOT_READY',retryable: true,status: 409,- retryAfter: '1',+ retryAfter: isParentWorking(parentJob) ? waitingRetry(envelope.receivedAt) : '1',deferWithoutAttempt: true,});}@@ -526,7 +538,7 @@ export function createBackgroundToolCompletionWakeupResolver({code: 'BACKGROUND_TOOL_RESULT_NOT_READY',retryable: true,status: 409,- retryAfter: '1',+ retryAfter: waitingRetry(envelope.receivedAt),deferWithoutAttempt: true,});};diff --git a/packages/api/src/agents/subagentCompletionWakeup.spec.ts b/packages/api/src/agents/subagentCompletionWakeup.spec.tsindex c8317a6906..16c54710c9 100644--- a/packages/api/src/agents/subagentCompletionWakeup.spec.ts+++ b/packages/api/src/agents/subagentCompletionWakeup.spec.ts@@ -1287,6 +1287,59 @@ describe('createSubagentCompletionWakeupResolver', () => {expect(snapshot.value.note).toContain('Do not infer that no other children ran');});+ it('backs off by waiting age while the child task is still running', async () => {+ const { methods } = resolverMethods();+ methods.getMessages.mockImplementation(async (filter: { conversationId: string }) =>+ filter.conversationId === 'conversation-1'+ ? [+ {+ messageId: 'response-1',+ parentMessageId: 'user-1',+ isCreatedByUser: false,+ createdAt: new Date(NOW - 30),+ },+ ]+ : [{ messageId: 'task-1:user', conversationId: 'thread-1', isCreatedByUser: true }],+ );+ const resolverAt = (offsetMs: number) =>+ createSubagentCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ now: () => NOW + offsetMs,+ });++ await expect(+ resolverAt(5_000)(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '5' });+ await expect(+ resolverAt(90_000)(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '9' });+ await expect(+ resolverAt(20 * 60_000)(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '60' });+ await expect(+ createSubagentCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => null,+ now: () => NOW + 20 * 60_000,+ getWaitMaxIntervalMs: () => 30_000,+ })(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '30' });+ });++ it('backs off by waiting age while the parent generation keeps running', async () => {+ const { methods } = resolverMethods();+ const resolve = createSubagentCompletionWakeupResolver({+ methods: methods as never,+ getGenerationJob: async () => ({ status: 'running' }),+ now: () => NOW + 150_000,+ });++ await expect(+ resolve(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),+ ).rejects.toMatchObject({ code: 'PARENT_NOT_READY', retryAfter: '15' });+ });+it('dead-letters a child whose process disappeared after the task timeout grace', async () => {const { methods } = resolverMethods();methods.getMessages.mockImplementation(async (filter: { conversationId: string }) =>diff --git a/packages/api/src/agents/subagentCompletionWakeup.ts b/packages/api/src/agents/subagentCompletionWakeup.tsindex 02240dcfb9..93285187bd 100644--- a/packages/api/src/agents/subagentCompletionWakeup.ts+++ b/packages/api/src/agents/subagentCompletionWakeup.ts@@ -9,6 +9,7 @@ import type { SubagentTaskWakeupRegistration } from './subagentThreads';import type { AgentContinueTriggerEnvelope } from './triggers/envelope';import type { AgentTriggerDispatchContext } from './triggers/dispatch';import type { AgentTriggerEnqueueOptions } from './triggers/delivery';+import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';import { boundedSubagentTaskResult } from './subagentTaskRouting';import { createAgentTriggerEnvelope } from './triggers/envelope';import { AgentTriggerExecutionError } from './triggers/host';@@ -81,6 +82,8 @@ export interface SubagentCompletionWakeupResolverDeps {methods: WakeupMethods;getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;now?: () => number;+ /** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */+ getWaitMaxIntervalMs?: () => number | undefined;}function payloadRegistration(@@ -139,6 +142,12 @@ function isParentActive(job: GenerationState | null): boolean {);}+/** A running or approval-paused parent can stay busy for hours; one that has+ * settled and is only finishing terminal persistence clears within moments. */+function isParentWorking(job: GenerationState | null): boolean {+ return job?.status === 'running' || job?.status === 'requires_action';+}+function sameTenant(actual: string | undefined, expected: string | undefined): boolean {return actual === expected;}@@ -579,9 +588,12 @@ export function createSubagentCompletionWakeupResolver({methods,getGenerationJob,now = Date.now,+ getWaitMaxIntervalMs,}: SubagentCompletionWakeupResolverDeps): NonNullable<AgentTriggerExecutionHostDeps['prepareContinue']> {+ const waitingRetry = (receivedAt: number): string =>+ waitingRetryAfter(receivedAt, now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);return async (envelope: AgentContinueTriggerEnvelope,context: AgentTriggerDispatchContext,@@ -616,7 +628,7 @@ export function createSubagentCompletionWakeupResolver({code: 'PARENT_NOT_READY',retryable: true,status: 409,- retryAfter: '1',+ retryAfter: isParentWorking(parentJob) ? waitingRetry(envelope.receivedAt) : '1',deferWithoutAttempt: true,});}@@ -700,7 +712,7 @@ export function createSubagentCompletionWakeupResolver({code: 'CHILD_NOT_READY',retryable: true,status: 409,- retryAfter: '1',+ retryAfter: waitingRetry(envelope.receivedAt),deferWithoutAttempt: true,});}diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.mdindex faef699ef5..875d797218 100644--- a/packages/api/src/agents/triggers/README.md+++ b/packages/api/src/agents/triggers/README.md@@ -99,6 +99,11 @@ experimental clustered startup:| `queuedTurnMaxIntervalMs` | 120000 | 30000–300000 || `maintenanceMaxIntervalMs` | 120000 | 30000–300000 || `deliveryMaxIntervalMs` | 15000 | 1000–300000 |+| `completionWaitMaxIntervalMs` | 60000 | 5000–300000 |++`completionWaitMaxIntervalMs` caps how long a background or subagent completion waits between+readiness checks while its result or parent turn is not ready. It backs off from 5 seconds by a tenth+of its age, so a long wait costs about one re-check per cap instead of one every five seconds.Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-datamigration is needed; optional activity reporting preserves the existing numeric/boolean results.diff --git a/packages/api/src/agents/triggers/backoff.spec.ts b/packages/api/src/agents/triggers/backoff.spec.tsnew file mode 100644index 0000000000..2fbed8f889--- /dev/null+++ b/packages/api/src/agents/triggers/backoff.spec.ts@@ -0,0 +1,28 @@+import { WAITING_RETRY_CAP_MS, WAITING_RETRY_FLOOR_MS, waitingRetryAfter } from './backoff';++describe('waitingRetryAfter', () => {+ const since = Date.parse('2026-09-24T12:00:00Z');+ const floor = String(WAITING_RETRY_FLOOR_MS / 1_000);++ it('re-checks a fresh wait at the engine floor', () => {+ expect(waitingRetryAfter(since, since)).toBe(floor);+ expect(waitingRetryAfter(since, since + 45_000)).toBe(floor);+ });++ it('waits a tenth of the elapsed wait once that exceeds the floor', () => {+ expect(waitingRetryAfter(since, since + 120_000)).toBe('12');+ expect(waitingRetryAfter(since, since + 5 * 60_000)).toBe('30');+ });++ it('never waits longer than the cap', () => {+ expect(waitingRetryAfter(since, since + 6 * 60 * 60_000)).toBe(+ String(WAITING_RETRY_CAP_MS / 1_000),+ );+ expect(waitingRetryAfter(since, since + 10 * 60_000, 15_000)).toBe('15');+ });++ it('treats a missing or future start as a fresh wait', () => {+ expect(waitingRetryAfter(Number.NaN, since)).toBe(floor);+ expect(waitingRetryAfter(since + 60_000, since)).toBe(floor);+ });+});diff --git a/packages/api/src/agents/triggers/backoff.ts b/packages/api/src/agents/triggers/backoff.tsnew file mode 100644index 0000000000..d5507b5b62--- /dev/null+++ b/packages/api/src/agents/triggers/backoff.ts@@ -0,0 +1,26 @@+/** How long a deferred delivery may wait before re-checking a condition nobody has+ * announced a change to. Matches the delivery engine's floor for readiness deferrals. */+export const WAITING_RETRY_FLOOR_MS = 5_000;+export const WAITING_RETRY_CAP_MS = 60_000;+const WAITING_RETRY_AGE_FRACTION = 0.1;++/**+ * Seconds until a waiting delivery re-checks readiness: a tenth of how long it+ * has waited, between the engine's five-second floor and the cap. Producers+ * expedite the delivery the moment its condition changes (a result becoming+ * durable, a generation settling), so this only bounds re-reads while nothing+ * has changed. Without it every pending delivery is re-claimed every five+ * seconds for as long as its tool runs or its conversation stays busy.+ */+export function waitingRetryAfter(+ waitingSinceMs: number,+ nowMs: number = Date.now(),+ capMs: number = WAITING_RETRY_CAP_MS,+): string {+ const waitedMs = Number.isFinite(waitingSinceMs) ? Math.max(0, nowMs - waitingSinceMs) : 0;+ const delayMs = Math.min(+ capMs,+ Math.max(WAITING_RETRY_FLOOR_MS, waitedMs * WAITING_RETRY_AGE_FRACTION),+ );+ return String(Math.ceil(delayMs / 1_000));+}diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.tsindex e00a600654..c5532179a6 100644--- a/packages/api/src/agents/triggers/engine.spec.ts+++ b/packages/api/src/agents/triggers/engine.spec.ts@@ -1,3 +1,4 @@+import { context, ROOT_CONTEXT } from '@opentelemetry/api';import type {AgentTriggerDeliveryFailure,AgentTriggerDeliveryRecord,@@ -91,6 +92,80 @@ describe('createAgentTriggerDeliveryEngine', () => {});});+ it.each([+ ['honours a longer requested wait', '30', 30_000],+ ['keeps the default re-check for a shorter request', '1', 5_000],+ ['keeps the default re-check without a request', undefined, 5_000],+ ])('%s for a readiness deferral', async (_label, retryAfter, expectedDelayMs) => {+ const store = storeWith();+ const dispatch = jest.fn(async () => {+ throw new AgentTriggerExecutionError('The parent generation has not settled yet.', {+ mode: 'continue',+ certainty: 'definite',+ code: 'PARENT_NOT_READY',+ retryable: true,+ deferWithoutAttempt: true,+ ...(retryAfter != null && { retryAfter }),+ });+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START, workerId: 'worker-1' },+ { concurrency: 1 },+ );++ await engine.runTick();++ expect(store.defer).toHaveBeenCalledWith(+ expect.objectContaining({+ availableAt: new Date(START.getTime() + expectedDelayMs),+ }),+ );+ expect(store.retry).not.toHaveBeenCalled();+ });++ it('runs each claim pass under the root trace context', async () => {+ const withContext = jest.spyOn(context, 'with');+ const store = storeWith({ claimNext: jest.fn(async () => null) });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: jest.fn(async () => successResult()), now: () => START },+ { concurrency: 1 },+ );++ try {+ await engine.runTick();+ expect(withContext).toHaveBeenCalledWith(ROOT_CONTEXT, expect.any(Function));+ expect(store.claimNext).toHaveBeenCalled();+ } finally {+ withContext.mockRestore();+ }+ });++ it('arms each poll timer under the root trace context', async () => {+ jest.useFakeTimers();+ const withContext = jest.spyOn(context, 'with');+ try {+ const store = storeWith({ claimNext: jest.fn(async () => null) });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: jest.fn(async () => successResult()), now: () => START },+ { concurrency: 1, tickMs: 1_000 },+ );+ engine.start();+ await jest.advanceTimersByTimeAsync(0);++ const timersArmed = withContext.mock.results.filter(+ (result, index) =>+ withContext.mock.calls[index][0] === ROOT_CONTEXT &&+ result.type === 'return' &&+ typeof (result.value as { unref?: unknown } | undefined)?.unref === 'function',+ );+ expect(timersArmed.length).toBeGreaterThan(0);+ await engine.stop();+ } finally {+ withContext.mockRestore();+ jest.useRealTimers();+ }+ });+it('persists generation identity when a bound continuation starts', async () => {const envelope = createAgentTriggerEnvelope({mode: 'continue',diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.tsindex 8fd6592a3b..ac5bf350e2 100644--- a/packages/api/src/agents/triggers/engine.ts+++ b/packages/api/src/agents/triggers/engine.ts@@ -1,4 +1,5 @@import { randomUUID } from 'node:crypto';+import { context, ROOT_CONTEXT } from '@opentelemetry/api';import { logger, runAsSystem } from '@librechat/data-schemas';import type { AgentTriggerExecutionResult } from './host';import { createAgentTriggerBatchEnvelope } from './batch';@@ -283,6 +284,29 @@ function normalizeFailure(failure: AgentTriggerDeliveryFailure): AgentTriggerDel};}+/** The delay an execution error asked for, as seconds or an HTTP date, clamped. */+function requestedRetryAfterMs(error: unknown, now: Date): number | undefined {+ if (!(error instanceof AgentTriggerExecutionError) || error.retryAfter == null) {+ return;+ }+ const seconds = Number(error.retryAfter);+ if (Number.isFinite(seconds) && seconds >= 0) {+ return Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS);+ }+ const absolute = Date.parse(error.retryAfter);+ if (Number.isFinite(absolute) && absolute > now.getTime()) {+ return Math.min(absolute - now.getTime(), MAX_RETRY_AFTER_MS);+ }+ return;+}++/** A readiness deferral may ask to wait longer than the default re-check, never+ * shorter: waiting producers back off by age, while every existing `retryAfter`+ * below the default keeps the cadence it has always had. */+function readinessDeferMs(error: unknown, now: Date): number {+ return Math.max(DEFAULT_DEFER_MS, requestedRetryAfterMs(error, now) ?? 0);+}+function retryAt(error: unknown,attempt: number,@@ -291,15 +315,9 @@ function retryAt(capMs: number,random: () => number,): Date {- if (error instanceof AgentTriggerExecutionError && error.retryAfter != null) {- const seconds = Number(error.retryAfter);- if (Number.isFinite(seconds) && seconds >= 0) {- return new Date(now.getTime() + Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS));- }- const absolute = Date.parse(error.retryAfter);- if (Number.isFinite(absolute) && absolute > now.getTime()) {- return new Date(Math.min(absolute, now.getTime() + MAX_RETRY_AFTER_MS));- }+ const requestedMs = requestedRetryAfterMs(error, now);+ if (requestedMs != null) {+ return new Date(now.getTime() + requestedMs);}const exponent = Math.min(attempt - 1, 30);const delay = Math.min(baseMs * 2 ** exponent, capMs);@@ -483,8 +501,12 @@ export function createAgentTriggerDeliveryEngine(deletionRejected ||runtimeNotReady) {- const delayMs =- error instanceof AgentTriggerDeliveryDeferredError ? error.delayMs : DEFAULT_DEFER_MS;+ let delayMs = DEFAULT_DEFER_MS;+ if (error instanceof AgentTriggerDeliveryDeferredError) {+ delayMs = error.delayMs;+ } else if (runtimeNotReady) {+ delayMs = readinessDeferMs(error, attemptedAt);+ }const availableAt = new Date(attemptedAt.getTime() + delayMs);noteEligibleAt(availableAt);const deferred = await deps.store.defer({@@ -693,7 +715,12 @@ export function createAgentTriggerDeliveryEngine(if (activeClaim != null) {return activeClaim;}- activeClaim = runAsSystem(runClaimPass)+ /** Claim passes are started from timers and from `wake()` calls made inside+ * request handlers. Run them under the root context so a pass never joins+ * whichever request happened to wake the engine — otherwise every later tick+ * inherits that request's trace for the life of the timer chain. */+ activeClaim = context+ .with(ROOT_CONTEXT, () => runAsSystem(runClaimPass)).then((result) => {/** Only a pass that confirmed an empty queue may advance the idle backoff: work* resets it, and a failed claim proves nothing, so it polls on at the base@@ -773,19 +800,23 @@ export function createAgentTriggerDeliveryEngine(if (eligibleDeadlinesMs.length > 0) {delay = Math.max(0, Math.min(delay, eligibleDeadlinesMs[0] - now().getTime()));}- timer = setTimeout(async () => {- if (stopped) {- return;- }- const nowMs = now().getTime();- while (eligibleDeadlinesMs.length > 0 && eligibleDeadlinesMs[0] <= nowMs) {- eligibleDeadlinesMs.shift();- }- await claimAvailable().catch((error) =>- logger.error('[agent-triggers] delivery claim pass failed:', error),- );- schedule();- }, delay);+ /** Created under the root context too: `schedule()` runs from `wake()` inside+ * request handlers, and each tick reschedules from its own callback. */+ timer = context.with(ROOT_CONTEXT, () =>+ setTimeout(async () => {+ if (stopped) {+ return;+ }+ const nowMs = now().getTime();+ while (eligibleDeadlinesMs.length > 0 && eligibleDeadlinesMs[0] <= nowMs) {+ eligibleDeadlinesMs.shift();+ }+ await claimAvailable().catch((error) =>+ logger.error('[agent-triggers] delivery claim pass failed:', error),+ );+ schedule();+ }, delay),+ );timer.unref();};diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsindex 8d571b0f92..1130b7f605 100644--- a/packages/api/src/agents/triggers/service.delivery.spec.ts+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -209,6 +209,18 @@ describe('durable agent trigger service', () => {await service.stop();});+ describe('completion wait configuration', () => {+ const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };++ it('exposes the configured completion wait cap', async () => {+ const service = createAgentTriggerService({ methods: deliveryMethods() });+ expect(service.getCompletionWaitMaxIntervalMs()).toBe(60_000);+ await service.initialize({ address, idlePolling: { completionWaitMaxIntervalMs: 20_000 } });+ expect(service.getCompletionWaitMaxIntervalMs()).toBe(20_000);+ await service.stop();+ });+ });+it('advertises ordinary completion but not detached-action capability without durable storage', async () => {const methods = deliveryMethods();const service = createAgentTriggerService({diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsindex a37e1821f3..f208336e57 100644--- a/packages/api/src/agents/triggers/service.ts+++ b/packages/api/src/agents/triggers/service.ts@@ -33,6 +33,7 @@ import { selfOriginFromAddress } from '../../app/origin';import { createAgentTriggerExecutionHost } from './host';import { parseAgentTriggerEnvelope } from './envelope';import { createIdleRecoveryLoop } from '../recovery';+import { WAITING_RETRY_CAP_MS } from './backoff';export const AGENT_TRIGGER_TOKEN_TTL = '60s';const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;@@ -48,6 +49,7 @@ export interface AgentTriggerServiceOptions {queuedTurnMaxIntervalMs?: number;maintenanceMaxIntervalMs?: number;deliveryMaxIntervalMs?: number;+ completionWaitMaxIntervalMs?: number;};}@@ -206,6 +208,8 @@ export interface AgentTriggerService {input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;getBackgroundCompletionResultBatchSize: () => number;+ /** Longest a waiting completion delivery re-checks readiness. */+ getCompletionWaitMaxIntervalMs: () => number;releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];drainUser: (userId: string) => Promise<void>;prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;@@ -323,6 +327,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}let boundOrigin: string | undefined;let backgroundCompletionResultBatchSize = 8;+ let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;let deliveryEngine: AgentTriggerDeliveryEngine | undefined;let initializePromise: Promise<void> | undefined;let purgeRecoveryPromise: Promise<boolean> | undefined;@@ -553,6 +558,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Areturn {initialize: (options = {}) => {backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;+ completionWaitMaxIntervalMs =+ options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;if (deps.methods == null || deliveryReady) {return Promise.resolve();@@ -728,6 +735,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Areturn getClaim == null ? null : getClaim(input);}),getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,+ getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,releaseBackgroundToolResultClaims: (input) =>runAsSystem(async () => {const release = requireMethods().releaseAgentBackgroundToolResultClaims;diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.tsindex 02676ab8ad..7ff73ec7a8 100644--- a/packages/data-provider/src/config.spec.ts+++ b/packages/data-provider/src/config.spec.ts@@ -682,12 +682,15 @@ describe('agent event runtime config', () => {deliveryMaxIntervalMs: 15_000,queuedTurnMaxIntervalMs: 120_000,maintenanceMaxIntervalMs: 120_000,+ completionWaitMaxIntervalMs: 60_000,});for (const [key, value] of [['deliveryMaxIntervalMs', 0],['queuedTurnMaxIntervalMs', 29_999],['maintenanceMaxIntervalMs', 300_001],['maintenanceMaxIntervalMs', 30_000.5],+ ['completionWaitMaxIntervalMs', 4_999],+ ['completionWaitMaxIntervalMs', 300_001],] as const) {expect(configSchema.safeParse({diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 698a5a6406..3acf59cd6b 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -1537,6 +1537,16 @@ export const agentsEndpointSchema = baseEndpointSchema.max(300_000).optional().default(120_000),+ /** Longest a background or subagent completion re-checks whether its+ * result and parent turn are ready. The events it waits on expedite it,+ * so this bounds missed signals rather than normal delivery latency. */+ completionWaitMaxIntervalMs: z+ .number()+ .int()+ .min(5_000)+ .max(300_000)+ .optional()+ .default(60_000),}).optional(),})diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.tsindex c2b6110930..3c6850d7b3 100644--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts@@ -2468,6 +2468,18 @@ describe('agent trigger delivery methods', () => {);});+ it("indexes a user's waiting deliveries for user-scoped readiness reads", async () => {+ const deliveryIndexes = await Delivery.collection.indexes();+ const userIndex = deliveryIndexes.find(+ (index) =>+ JSON.stringify(index.key) === JSON.stringify({ user: 1, status: 1, availableAt: 1 }),+ );+ expect(userIndex).toBeDefined();+ /** A sparse index would skip every row the expedite and listing reads need. */+ expect(userIndex?.sparse).toBeUndefined();+ expect(userIndex?.partialFilterExpression).toBeUndefined();+ });+it('publishes an idempotent replay on its persisted ordering lane', async () => {const user = new mongoose.Types.ObjectId();const input = enqueueInput({ user, orderingKey: 'original-lane' });diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.tsindex 1de097809d..2d08f6d14c 100644--- a/packages/data-schemas/src/schema/triggerDelivery.ts+++ b/packages/data-schemas/src/schema/triggerDelivery.ts@@ -253,6 +253,8 @@ triggerDeliverySchema.index({ sparse: true },);triggerDeliverySchema.index({ status: 1, updatedAt: -1 });+/** One user's waiting deliveries, read when their readiness changes. */+triggerDeliverySchema.index({ user: 1, status: 1, availableAt: 1 });triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });