Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🐢 fix: Back Off Waiting Completion Wake-ups and Keep Their Trace Out of Requests (#16349)

Danny Avila3788fe05d92e
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index 7b58b54a9e..329bfd4635 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -24,11 +24,13 @@ const getGenerationAdmissionEvidence = (userId, clientRequestId, streamId, conve
const subagentCompletionAdapter = createSubagentCompletionWakeupResolver({
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
+ getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),
});
const backgroundToolCompletionAdapter = createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
getResultBatchSize: () => service.getBackgroundCompletionResultBatchSize(),
+ getWaitMaxIntervalMs: () => service.getCompletionWaitMaxIntervalMs(),
});
const eventActorAdapter = createAgentEventContinueResolver({
methods,
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
index 1e87d6a2f5..ceb95fe540 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
@@ -756,6 +756,88 @@ describe('background tool completion wakeups', () => {
expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
});
+ it('backs off by waiting age while the invoking generation keeps running', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => ({ status: 'running' }),
+ });
+ const deliveryEnvelope = await envelope();
+
+ jest.setSystemTime(NOW + 120_000);
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'PARENT_NOT_READY', retryAfter: '12', deferWithoutAttempt: true },
+ );
+ jest.setSystemTime(NOW + 6 * 60 * 60_000);
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'PARENT_NOT_READY', retryAfter: '60' },
+ );
+ });
+
+ it('caps the waiting backoff at the configured interval', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => ({ status: 'running' }),
+ getWaitMaxIntervalMs: () => 20_000,
+ });
+ const deliveryEnvelope = await envelope();
+
+ jest.setSystemTime(NOW + 6 * 60 * 60_000);
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'PARENT_NOT_READY', retryAfter: '20' },
+ );
+ });
+
+ it('keeps backing off while the parent is paused for approval', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => ({ status: 'requires_action' }),
+ });
+ const deliveryEnvelope = await envelope();
+
+ jest.setSystemTime(NOW + 60 * 60_000);
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'PARENT_NOT_READY', retryAfter: '60' },
+ );
+ });
+
+ it('re-checks within a second once the parent has settled and only persistence remains', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => ({
+ status: 'complete',
+ metadata: { terminalPersistencePending: true },
+ }),
+ });
+ const deliveryEnvelope = await envelope();
+
+ jest.setSystemTime(NOW + 10 * 60_000);
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'PARENT_NOT_READY', retryAfter: '1' },
+ );
+ });
+
+ it('backs off by waiting age while the tool result is not durable yet', async () => {
+ const { methods } = resolverMethods();
+ methods.claimBackgroundToolResults.mockResolvedValue({ status: 'missing', results: [] });
+ const resolve = createBackgroundToolCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ });
+ const deliveryEnvelope = await envelope();
+
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'BACKGROUND_TOOL_RESULT_NOT_READY', retryAfter: '5' },
+ );
+ jest.setSystemTime(NOW + 5 * 60_000);
+ await expect(resolve(deliveryEnvelope, { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
+ { code: 'BACKGROUND_TOOL_RESULT_NOT_READY', retryAfter: '30' },
+ );
+ });
+
it('does not manufacture terminal evidence from wall-clock age', async () => {
const { methods } = resolverMethods();
methods.claimBackgroundToolResults.mockResolvedValue({ status: 'missing', results: [] });
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts
index bfdb9f21d3..1ca6fca26c 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts
@@ -21,6 +21,7 @@ import type {
import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
import type { AgentTriggerDispatchContext } from './triggers/dispatch';
import type { AgentTriggerEnqueueOptions } from './triggers/delivery';
+import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';
import { BACKGROUND_TOOL_PRODUCER_LEASE_MS } from './backgroundCompletion';
import { createAgentTriggerEnvelope } from './triggers/envelope';
import { AgentTriggerExecutionError } from './triggers/host';
@@ -94,6 +95,8 @@ export interface BackgroundToolCompletionWakeupResolverDeps {
methods: WakeupMethods;
getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
getResultBatchSize?: () => number | undefined;
+ /** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */
+ getWaitMaxIntervalMs?: () => number | undefined;
}
function executionError(
@@ -152,6 +155,12 @@ function isParentActive(job: GenerationState | null): boolean {
);
}
+/** A running or approval-paused parent can stay busy for hours; one that has
+ * settled and is only finishing terminal persistence clears within moments. */
+function isParentWorking(job: GenerationState | null): boolean {
+ return job?.status === 'running' || job?.status === 'requires_action';
+}
+
function timestamp(message: Pick<IMessage, 'createdAt'>): number {
const value = message.createdAt;
if (value instanceof Date) {
@@ -252,9 +261,12 @@ export function createBackgroundToolCompletionWakeupResolver({
methods,
getGenerationJob,
getResultBatchSize,
+ getWaitMaxIntervalMs,
}: BackgroundToolCompletionWakeupResolverDeps): NonNullable<
AgentTriggerExecutionHostDeps['prepareContinue']
> {
+ const waitingRetry = (receivedAt: number): string =>
+ waitingRetryAfter(receivedAt, Date.now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);
return async (
envelope: AgentContinueTriggerEnvelope,
context: AgentTriggerDispatchContext,
@@ -288,7 +300,7 @@ export function createBackgroundToolCompletionWakeupResolver({
code: 'PARENT_NOT_READY',
retryable: true,
status: 409,
- retryAfter: '1',
+ retryAfter: isParentWorking(parentJob) ? waitingRetry(envelope.receivedAt) : '1',
deferWithoutAttempt: true,
});
}
@@ -526,7 +538,7 @@ export function createBackgroundToolCompletionWakeupResolver({
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
retryable: true,
status: 409,
- retryAfter: '1',
+ retryAfter: waitingRetry(envelope.receivedAt),
deferWithoutAttempt: true,
});
};
diff --git a/packages/api/src/agents/subagentCompletionWakeup.spec.ts b/packages/api/src/agents/subagentCompletionWakeup.spec.ts
index c8317a6906..16c54710c9 100644
--- a/packages/api/src/agents/subagentCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/subagentCompletionWakeup.spec.ts
@@ -1287,6 +1287,59 @@ describe('createSubagentCompletionWakeupResolver', () => {
expect(snapshot.value.note).toContain('Do not infer that no other children ran');
});
+ it('backs off by waiting age while the child task is still running', async () => {
+ const { methods } = resolverMethods();
+ methods.getMessages.mockImplementation(async (filter: { conversationId: string }) =>
+ filter.conversationId === 'conversation-1'
+ ? [
+ {
+ messageId: 'response-1',
+ parentMessageId: 'user-1',
+ isCreatedByUser: false,
+ createdAt: new Date(NOW - 30),
+ },
+ ]
+ : [{ messageId: 'task-1:user', conversationId: 'thread-1', isCreatedByUser: true }],
+ );
+ const resolverAt = (offsetMs: number) =>
+ createSubagentCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ now: () => NOW + offsetMs,
+ });
+
+ await expect(
+ resolverAt(5_000)(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '5' });
+ await expect(
+ resolverAt(90_000)(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '9' });
+ await expect(
+ resolverAt(20 * 60_000)(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '60' });
+ await expect(
+ createSubagentCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => null,
+ now: () => NOW + 20 * 60_000,
+ getWaitMaxIntervalMs: () => 30_000,
+ })(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
+ ).rejects.toMatchObject({ code: 'CHILD_NOT_READY', retryAfter: '30' });
+ });
+
+ it('backs off by waiting age while the parent generation keeps running', async () => {
+ const { methods } = resolverMethods();
+ const resolve = createSubagentCompletionWakeupResolver({
+ methods: methods as never,
+ getGenerationJob: async () => ({ status: 'running' }),
+ now: () => NOW + 150_000,
+ });
+
+ await expect(
+ resolve(wakeupEnvelope(), { idempotencyKey: 'trigger_claim_1' } as never),
+ ).rejects.toMatchObject({ code: 'PARENT_NOT_READY', retryAfter: '15' });
+ });
+
it('dead-letters a child whose process disappeared after the task timeout grace', async () => {
const { methods } = resolverMethods();
methods.getMessages.mockImplementation(async (filter: { conversationId: string }) =>
diff --git a/packages/api/src/agents/subagentCompletionWakeup.ts b/packages/api/src/agents/subagentCompletionWakeup.ts
index 02240dcfb9..93285187bd 100644
--- a/packages/api/src/agents/subagentCompletionWakeup.ts
+++ b/packages/api/src/agents/subagentCompletionWakeup.ts
@@ -9,6 +9,7 @@ import type { SubagentTaskWakeupRegistration } from './subagentThreads';
import type { AgentContinueTriggerEnvelope } from './triggers/envelope';
import type { AgentTriggerDispatchContext } from './triggers/dispatch';
import type { AgentTriggerEnqueueOptions } from './triggers/delivery';
+import { WAITING_RETRY_CAP_MS, waitingRetryAfter } from './triggers/backoff';
import { boundedSubagentTaskResult } from './subagentTaskRouting';
import { createAgentTriggerEnvelope } from './triggers/envelope';
import { AgentTriggerExecutionError } from './triggers/host';
@@ -81,6 +82,8 @@ export interface SubagentCompletionWakeupResolverDeps {
methods: WakeupMethods;
getGenerationJob: (conversationId: string) => Promise<GenerationState | null>;
now?: () => number;
+ /** Longest a waiting delivery re-checks readiness; the backoff default otherwise. */
+ getWaitMaxIntervalMs?: () => number | undefined;
}
function payloadRegistration(
@@ -139,6 +142,12 @@ function isParentActive(job: GenerationState | null): boolean {
);
}
+/** A running or approval-paused parent can stay busy for hours; one that has
+ * settled and is only finishing terminal persistence clears within moments. */
+function isParentWorking(job: GenerationState | null): boolean {
+ return job?.status === 'running' || job?.status === 'requires_action';
+}
+
function sameTenant(actual: string | undefined, expected: string | undefined): boolean {
return actual === expected;
}
@@ -579,9 +588,12 @@ export function createSubagentCompletionWakeupResolver({
methods,
getGenerationJob,
now = Date.now,
+ getWaitMaxIntervalMs,
}: SubagentCompletionWakeupResolverDeps): NonNullable<
AgentTriggerExecutionHostDeps['prepareContinue']
> {
+ const waitingRetry = (receivedAt: number): string =>
+ waitingRetryAfter(receivedAt, now(), getWaitMaxIntervalMs?.() ?? WAITING_RETRY_CAP_MS);
return async (
envelope: AgentContinueTriggerEnvelope,
context: AgentTriggerDispatchContext,
@@ -616,7 +628,7 @@ export function createSubagentCompletionWakeupResolver({
code: 'PARENT_NOT_READY',
retryable: true,
status: 409,
- retryAfter: '1',
+ retryAfter: isParentWorking(parentJob) ? waitingRetry(envelope.receivedAt) : '1',
deferWithoutAttempt: true,
});
}
@@ -700,7 +712,7 @@ export function createSubagentCompletionWakeupResolver({
code: 'CHILD_NOT_READY',
retryable: true,
status: 409,
- retryAfter: '1',
+ retryAfter: waitingRetry(envelope.receivedAt),
deferWithoutAttempt: true,
});
}
diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md
index faef699ef5..875d797218 100644
--- a/packages/api/src/agents/triggers/README.md
+++ b/packages/api/src/agents/triggers/README.md
@@ -99,6 +99,11 @@ experimental clustered startup:
| `queuedTurnMaxIntervalMs` | 120000 | 30000–300000 |
| `maintenanceMaxIntervalMs` | 120000 | 30000–300000 |
| `deliveryMaxIntervalMs` | 15000 | 1000–300000 |
+| `completionWaitMaxIntervalMs` | 60000 | 5000–300000 |
+
+`completionWaitMaxIntervalMs` caps how long a background or subagent completion waits between
+readiness checks while its result or parent turn is not ready. It backs off from 5 seconds by a tenth
+of its age, so a long wait costs about one re-check per cap instead of one every five seconds.
Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data
migration is needed; optional activity reporting preserves the existing numeric/boolean results.
diff --git a/packages/api/src/agents/triggers/backoff.spec.ts b/packages/api/src/agents/triggers/backoff.spec.ts
new file mode 100644
index 0000000000..2fbed8f889
--- /dev/null
+++ b/packages/api/src/agents/triggers/backoff.spec.ts
@@ -0,0 +1,28 @@
+import { WAITING_RETRY_CAP_MS, WAITING_RETRY_FLOOR_MS, waitingRetryAfter } from './backoff';
+
+describe('waitingRetryAfter', () => {
+ const since = Date.parse('2026-09-24T12:00:00Z');
+ const floor = String(WAITING_RETRY_FLOOR_MS / 1_000);
+
+ it('re-checks a fresh wait at the engine floor', () => {
+ expect(waitingRetryAfter(since, since)).toBe(floor);
+ expect(waitingRetryAfter(since, since + 45_000)).toBe(floor);
+ });
+
+ it('waits a tenth of the elapsed wait once that exceeds the floor', () => {
+ expect(waitingRetryAfter(since, since + 120_000)).toBe('12');
+ expect(waitingRetryAfter(since, since + 5 * 60_000)).toBe('30');
+ });
+
+ it('never waits longer than the cap', () => {
+ expect(waitingRetryAfter(since, since + 6 * 60 * 60_000)).toBe(
+ String(WAITING_RETRY_CAP_MS / 1_000),
+ );
+ expect(waitingRetryAfter(since, since + 10 * 60_000, 15_000)).toBe('15');
+ });
+
+ it('treats a missing or future start as a fresh wait', () => {
+ expect(waitingRetryAfter(Number.NaN, since)).toBe(floor);
+ expect(waitingRetryAfter(since + 60_000, since)).toBe(floor);
+ });
+});
diff --git a/packages/api/src/agents/triggers/backoff.ts b/packages/api/src/agents/triggers/backoff.ts
new file mode 100644
index 0000000000..d5507b5b62
--- /dev/null
+++ b/packages/api/src/agents/triggers/backoff.ts
@@ -0,0 +1,26 @@
+/** How long a deferred delivery may wait before re-checking a condition nobody has
+ * announced a change to. Matches the delivery engine's floor for readiness deferrals. */
+export const WAITING_RETRY_FLOOR_MS = 5_000;
+export const WAITING_RETRY_CAP_MS = 60_000;
+const WAITING_RETRY_AGE_FRACTION = 0.1;
+
+/**
+ * Seconds until a waiting delivery re-checks readiness: a tenth of how long it
+ * has waited, between the engine's five-second floor and the cap. Producers
+ * expedite the delivery the moment its condition changes (a result becoming
+ * durable, a generation settling), so this only bounds re-reads while nothing
+ * has changed. Without it every pending delivery is re-claimed every five
+ * seconds for as long as its tool runs or its conversation stays busy.
+ */
+export function waitingRetryAfter(
+ waitingSinceMs: number,
+ nowMs: number = Date.now(),
+ capMs: number = WAITING_RETRY_CAP_MS,
+): string {
+ const waitedMs = Number.isFinite(waitingSinceMs) ? Math.max(0, nowMs - waitingSinceMs) : 0;
+ const delayMs = Math.min(
+ capMs,
+ Math.max(WAITING_RETRY_FLOOR_MS, waitedMs * WAITING_RETRY_AGE_FRACTION),
+ );
+ return String(Math.ceil(delayMs / 1_000));
+}
diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts
index e00a600654..c5532179a6 100644
--- a/packages/api/src/agents/triggers/engine.spec.ts
+++ b/packages/api/src/agents/triggers/engine.spec.ts
@@ -1,3 +1,4 @@
+import { context, ROOT_CONTEXT } from '@opentelemetry/api';
import type {
AgentTriggerDeliveryFailure,
AgentTriggerDeliveryRecord,
@@ -91,6 +92,80 @@ describe('createAgentTriggerDeliveryEngine', () => {
});
});
+ it.each([
+ ['honours a longer requested wait', '30', 30_000],
+ ['keeps the default re-check for a shorter request', '1', 5_000],
+ ['keeps the default re-check without a request', undefined, 5_000],
+ ])('%s for a readiness deferral', async (_label, retryAfter, expectedDelayMs) => {
+ const store = storeWith();
+ const dispatch = jest.fn(async () => {
+ throw new AgentTriggerExecutionError('The parent generation has not settled yet.', {
+ mode: 'continue',
+ certainty: 'definite',
+ code: 'PARENT_NOT_READY',
+ retryable: true,
+ deferWithoutAttempt: true,
+ ...(retryAfter != null && { retryAfter }),
+ });
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START, workerId: 'worker-1' },
+ { concurrency: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.defer).toHaveBeenCalledWith(
+ expect.objectContaining({
+ availableAt: new Date(START.getTime() + expectedDelayMs),
+ }),
+ );
+ expect(store.retry).not.toHaveBeenCalled();
+ });
+
+ it('runs each claim pass under the root trace context', async () => {
+ const withContext = jest.spyOn(context, 'with');
+ const store = storeWith({ claimNext: jest.fn(async () => null) });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: jest.fn(async () => successResult()), now: () => START },
+ { concurrency: 1 },
+ );
+
+ try {
+ await engine.runTick();
+ expect(withContext).toHaveBeenCalledWith(ROOT_CONTEXT, expect.any(Function));
+ expect(store.claimNext).toHaveBeenCalled();
+ } finally {
+ withContext.mockRestore();
+ }
+ });
+
+ it('arms each poll timer under the root trace context', async () => {
+ jest.useFakeTimers();
+ const withContext = jest.spyOn(context, 'with');
+ try {
+ const store = storeWith({ claimNext: jest.fn(async () => null) });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: jest.fn(async () => successResult()), now: () => START },
+ { concurrency: 1, tickMs: 1_000 },
+ );
+ engine.start();
+ await jest.advanceTimersByTimeAsync(0);
+
+ const timersArmed = withContext.mock.results.filter(
+ (result, index) =>
+ withContext.mock.calls[index][0] === ROOT_CONTEXT &&
+ result.type === 'return' &&
+ typeof (result.value as { unref?: unknown } | undefined)?.unref === 'function',
+ );
+ expect(timersArmed.length).toBeGreaterThan(0);
+ await engine.stop();
+ } finally {
+ withContext.mockRestore();
+ jest.useRealTimers();
+ }
+ });
+
it('persists generation identity when a bound continuation starts', async () => {
const envelope = createAgentTriggerEnvelope({
mode: 'continue',
diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts
index 8fd6592a3b..ac5bf350e2 100644
--- a/packages/api/src/agents/triggers/engine.ts
+++ b/packages/api/src/agents/triggers/engine.ts
@@ -1,4 +1,5 @@
import { randomUUID } from 'node:crypto';
+import { context, ROOT_CONTEXT } from '@opentelemetry/api';
import { logger, runAsSystem } from '@librechat/data-schemas';
import type { AgentTriggerExecutionResult } from './host';
import { createAgentTriggerBatchEnvelope } from './batch';
@@ -283,6 +284,29 @@ function normalizeFailure(failure: AgentTriggerDeliveryFailure): AgentTriggerDel
};
}
+/** The delay an execution error asked for, as seconds or an HTTP date, clamped. */
+function requestedRetryAfterMs(error: unknown, now: Date): number | undefined {
+ if (!(error instanceof AgentTriggerExecutionError) || error.retryAfter == null) {
+ return;
+ }
+ const seconds = Number(error.retryAfter);
+ if (Number.isFinite(seconds) && seconds >= 0) {
+ return Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS);
+ }
+ const absolute = Date.parse(error.retryAfter);
+ if (Number.isFinite(absolute) && absolute > now.getTime()) {
+ return Math.min(absolute - now.getTime(), MAX_RETRY_AFTER_MS);
+ }
+ return;
+}
+
+/** A readiness deferral may ask to wait longer than the default re-check, never
+ * shorter: waiting producers back off by age, while every existing `retryAfter`
+ * below the default keeps the cadence it has always had. */
+function readinessDeferMs(error: unknown, now: Date): number {
+ return Math.max(DEFAULT_DEFER_MS, requestedRetryAfterMs(error, now) ?? 0);
+}
+
function retryAt(
error: unknown,
attempt: number,
@@ -291,15 +315,9 @@ function retryAt(
capMs: number,
random: () => number,
): Date {
- if (error instanceof AgentTriggerExecutionError && error.retryAfter != null) {
- const seconds = Number(error.retryAfter);
- if (Number.isFinite(seconds) && seconds >= 0) {
- return new Date(now.getTime() + Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS));
- }
- const absolute = Date.parse(error.retryAfter);
- if (Number.isFinite(absolute) && absolute > now.getTime()) {
- return new Date(Math.min(absolute, now.getTime() + MAX_RETRY_AFTER_MS));
- }
+ const requestedMs = requestedRetryAfterMs(error, now);
+ if (requestedMs != null) {
+ return new Date(now.getTime() + requestedMs);
}
const exponent = Math.min(attempt - 1, 30);
const delay = Math.min(baseMs * 2 ** exponent, capMs);
@@ -483,8 +501,12 @@ export function createAgentTriggerDeliveryEngine(
deletionRejected ||
runtimeNotReady
) {
- const delayMs =
- error instanceof AgentTriggerDeliveryDeferredError ? error.delayMs : DEFAULT_DEFER_MS;
+ let delayMs = DEFAULT_DEFER_MS;
+ if (error instanceof AgentTriggerDeliveryDeferredError) {
+ delayMs = error.delayMs;
+ } else if (runtimeNotReady) {
+ delayMs = readinessDeferMs(error, attemptedAt);
+ }
const availableAt = new Date(attemptedAt.getTime() + delayMs);
noteEligibleAt(availableAt);
const deferred = await deps.store.defer({
@@ -693,7 +715,12 @@ export function createAgentTriggerDeliveryEngine(
if (activeClaim != null) {
return activeClaim;
}
- activeClaim = runAsSystem(runClaimPass)
+ /** Claim passes are started from timers and from `wake()` calls made inside
+ * request handlers. Run them under the root context so a pass never joins
+ * whichever request happened to wake the engine — otherwise every later tick
+ * inherits that request's trace for the life of the timer chain. */
+ activeClaim = context
+ .with(ROOT_CONTEXT, () => runAsSystem(runClaimPass))
.then((result) => {
/** Only a pass that confirmed an empty queue may advance the idle backoff: work
* resets it, and a failed claim proves nothing, so it polls on at the base
@@ -773,19 +800,23 @@ export function createAgentTriggerDeliveryEngine(
if (eligibleDeadlinesMs.length > 0) {
delay = Math.max(0, Math.min(delay, eligibleDeadlinesMs[0] - now().getTime()));
}
- timer = setTimeout(async () => {
- if (stopped) {
- return;
- }
- const nowMs = now().getTime();
- while (eligibleDeadlinesMs.length > 0 && eligibleDeadlinesMs[0] <= nowMs) {
- eligibleDeadlinesMs.shift();
- }
- await claimAvailable().catch((error) =>
- logger.error('[agent-triggers] delivery claim pass failed:', error),
- );
- schedule();
- }, delay);
+ /** Created under the root context too: `schedule()` runs from `wake()` inside
+ * request handlers, and each tick reschedules from its own callback. */
+ timer = context.with(ROOT_CONTEXT, () =>
+ setTimeout(async () => {
+ if (stopped) {
+ return;
+ }
+ const nowMs = now().getTime();
+ while (eligibleDeadlinesMs.length > 0 && eligibleDeadlinesMs[0] <= nowMs) {
+ eligibleDeadlinesMs.shift();
+ }
+ await claimAvailable().catch((error) =>
+ logger.error('[agent-triggers] delivery claim pass failed:', error),
+ );
+ schedule();
+ }, delay),
+ );
timer.unref();
};
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index 8d571b0f92..1130b7f605 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -209,6 +209,18 @@ describe('durable agent trigger service', () => {
await service.stop();
});
+ describe('completion wait configuration', () => {
+ const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };
+
+ it('exposes the configured completion wait cap', async () => {
+ const service = createAgentTriggerService({ methods: deliveryMethods() });
+ expect(service.getCompletionWaitMaxIntervalMs()).toBe(60_000);
+ await service.initialize({ address, idlePolling: { completionWaitMaxIntervalMs: 20_000 } });
+ expect(service.getCompletionWaitMaxIntervalMs()).toBe(20_000);
+ await service.stop();
+ });
+ });
+
it('advertises ordinary completion but not detached-action capability without durable storage', async () => {
const methods = deliveryMethods();
const service = createAgentTriggerService({
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index a37e1821f3..f208336e57 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -33,6 +33,7 @@ import { selfOriginFromAddress } from '../../app/origin';
import { createAgentTriggerExecutionHost } from './host';
import { parseAgentTriggerEnvelope } from './envelope';
import { createIdleRecoveryLoop } from '../recovery';
+import { WAITING_RETRY_CAP_MS } from './backoff';
export const AGENT_TRIGGER_TOKEN_TTL = '60s';
const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;
@@ -48,6 +49,7 @@ export interface AgentTriggerServiceOptions {
queuedTurnMaxIntervalMs?: number;
maintenanceMaxIntervalMs?: number;
deliveryMaxIntervalMs?: number;
+ completionWaitMaxIntervalMs?: number;
};
}
@@ -206,6 +208,8 @@ export interface AgentTriggerService {
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
getBackgroundCompletionResultBatchSize: () => number;
+ /** Longest a waiting completion delivery re-checks readiness. */
+ getCompletionWaitMaxIntervalMs: () => number;
releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
drainUser: (userId: string) => Promise<void>;
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
@@ -323,6 +327,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
let boundOrigin: string | undefined;
let backgroundCompletionResultBatchSize = 8;
+ let completionWaitMaxIntervalMs = WAITING_RETRY_CAP_MS;
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
let initializePromise: Promise<void> | undefined;
let purgeRecoveryPromise: Promise<boolean> | undefined;
@@ -553,6 +558,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return {
initialize: (options = {}) => {
backgroundCompletionResultBatchSize = options.completionResultBatchSize ?? 8;
+ completionWaitMaxIntervalMs =
+ options.idlePolling?.completionWaitMaxIntervalMs ?? WAITING_RETRY_CAP_MS;
boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
if (deps.methods == null || deliveryReady) {
return Promise.resolve();
@@ -728,6 +735,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
return getClaim == null ? null : getClaim(input);
}),
getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,
+ getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,
releaseBackgroundToolResultClaims: (input) =>
runAsSystem(async () => {
const release = requireMethods().releaseAgentBackgroundToolResultClaims;
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index 02676ab8ad..7ff73ec7a8 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -682,12 +682,15 @@ describe('agent event runtime config', () => {
deliveryMaxIntervalMs: 15_000,
queuedTurnMaxIntervalMs: 120_000,
maintenanceMaxIntervalMs: 120_000,
+ completionWaitMaxIntervalMs: 60_000,
});
for (const [key, value] of [
['deliveryMaxIntervalMs', 0],
['queuedTurnMaxIntervalMs', 29_999],
['maintenanceMaxIntervalMs', 300_001],
['maintenanceMaxIntervalMs', 30_000.5],
+ ['completionWaitMaxIntervalMs', 4_999],
+ ['completionWaitMaxIntervalMs', 300_001],
] as const) {
expect(
configSchema.safeParse({
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 698a5a6406..3acf59cd6b 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -1537,6 +1537,16 @@ export const agentsEndpointSchema = baseEndpointSchema
.max(300_000)
.optional()
.default(120_000),
+ /** Longest a background or subagent completion re-checks whether its
+ * result and parent turn are ready. The events it waits on expedite it,
+ * so this bounds missed signals rather than normal delivery latency. */
+ completionWaitMaxIntervalMs: z
+ .number()
+ .int()
+ .min(5_000)
+ .max(300_000)
+ .optional()
+ .default(60_000),
})
.optional(),
})
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index c2b6110930..3c6850d7b3 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -2468,6 +2468,18 @@ describe('agent trigger delivery methods', () => {
);
});
+ it("indexes a user's waiting deliveries for user-scoped readiness reads", async () => {
+ const deliveryIndexes = await Delivery.collection.indexes();
+ const userIndex = deliveryIndexes.find(
+ (index) =>
+ JSON.stringify(index.key) === JSON.stringify({ user: 1, status: 1, availableAt: 1 }),
+ );
+ expect(userIndex).toBeDefined();
+ /** A sparse index would skip every row the expedite and listing reads need. */
+ expect(userIndex?.sparse).toBeUndefined();
+ expect(userIndex?.partialFilterExpression).toBeUndefined();
+ });
+
it('publishes an idempotent replay on its persisted ordering lane', async () => {
const user = new mongoose.Types.ObjectId();
const input = enqueueInput({ user, orderingKey: 'original-lane' });
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
index 1de097809d..2d08f6d14c 100644
--- a/packages/data-schemas/src/schema/triggerDelivery.ts
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -253,6 +253,8 @@ triggerDeliverySchema.index(
{ sparse: true },
);
triggerDeliverySchema.index({ status: 1, updatedAt: -1 });
+/** One user's waiting deliveries, read when their readiness changes. */
+triggerDeliverySchema.index({ user: 1, status: 1, availableAt: 1 });
triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });
triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });