Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

⚡ fix: Deliver Waiting Completion Wake-ups the Moment They Are Ready (#16339)

Danny Avila7238be3e5ea1
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index 329bfd4635..f8944c6ea8 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -53,6 +53,7 @@ service = createAgentTriggerService({
),
isPrincipalActive: methods.isAgentTriggerPrincipalActive,
supportsDetachedActionCompletion: () => GenerationJobManager.supportsDetachedAgentEventActions,
+ subscribeGenerationSettled: (listener) => GenerationJobManager.onGenerationSettled(listener),
settleSourceBeforeDeadLetter: queuedTurnLifecycle.settleBeforeDeadLetter,
prepareContinue: createAgentContinuationResolver({
eventActor: eventActorAdapter,
@@ -92,6 +93,7 @@ module.exports = {
retireAgentTrigger: service.retire,
renewAgentTriggerProducerLease: service.renewProducerLease,
persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
+ expediteCompletionWakeups: service.expediteCompletionWakeups,
getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
drainAgentTriggerDeliveriesForUser: service.drainUser,
diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.js
index 8ca1cae6ef..da93d98242 100644
--- a/api/server/services/Endpoints/agents/backgroundCompletion.js
+++ b/api/server/services/Endpoints/agents/backgroundCompletion.js
@@ -11,6 +11,7 @@ const {
releaseAgentBackgroundToolResultClaims,
renewAgentTriggerProducerLease,
retireAgentTrigger,
+ expediteCompletionWakeups,
} = require('../../Agents/triggers');
const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeupHandler(
@@ -19,6 +20,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
renewAgentTriggerProducerLease,
(deliveryKey, sourceId, result) =>
persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
+ (deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }),
);
function createBackgroundToolResultPersistence({ req, updateToolCallResult }) {
diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.js b/api/server/services/Endpoints/agents/subagentThreadStore.js
index b33b7df541..d98f0f275f 100644
--- a/api/server/services/Endpoints/agents/subagentThreadStore.js
+++ b/api/server/services/Endpoints/agents/subagentThreadStore.js
@@ -12,7 +12,7 @@ const {
SubagentActivityStream,
} = require('@librechat/api');
const db = require('~/models');
-const { enqueueAgentTrigger } = require('../../Agents/triggers');
+const { enqueueAgentTrigger, expediteCompletionWakeups } = require('../../Agents/triggers');
const GENERATION_DRAIN_TIMEOUT_MS = 45_000;
const GENERATION_DRAIN_POLL_MS = 100;
@@ -78,6 +78,8 @@ const subagentThreadTaskStore = createSubagentThreadTaskStore(
releaseOwnerAdmission: db.releaseSubagentAdmission,
cancelUnroutedTask: cancelUnroutedGeneration,
onTaskPrepared: completionWakeupHandler,
+ onTaskSettled: (userId, conversationId, taskIds) =>
+ expediteCompletionWakeups({ user: userId, conversationId, taskIds }),
},
);
diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js
index bae8a2cdeb..a7d2609468 100644
--- a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js
+++ b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js
@@ -45,6 +45,7 @@ jest.mock('~/models', () => ({
jest.mock('../../Agents/triggers', () => ({
enqueueAgentTrigger: jest.fn(),
+ expediteCompletionWakeups: jest.fn(),
}));
const {
@@ -80,6 +81,16 @@ describe('subagent thread Redis lifecycle', () => {
expect(mockCompletionWakeupHandler).toHaveBeenCalledWith({ taskId: 'task-1' });
});
+ it('expedites only the settled task identities in their parent conversation', () => {
+ const { expediteCompletionWakeups } = require('../../Agents/triggers');
+ taskStoreOptions.onTaskSettled('user-1', 'parent-1', ['task-1', 'recovered-task']);
+ expect(expediteCompletionWakeups).toHaveBeenCalledWith({
+ user: 'user-1',
+ conversationId: 'parent-1',
+ taskIds: ['task-1', 'recovered-task'],
+ });
+ });
+
it('registers local task-store quiescence independently of optional Redis setup', () => {
expect(taskStoreShutdownRegistration).toEqual([
'subagent task store',
diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts
index 8ef1d74cca..02d57cb7d8 100644
--- a/packages/api/src/agents/backgroundCompletion.ts
+++ b/packages/api/src/agents/backgroundCompletion.ts
@@ -42,6 +42,9 @@ export interface BackgroundToolWakeupAdmission {
* Manual polling requires an atomic unclaimed-only transition: once a
* resolver owns a lease, its prepared continuation cannot be cancelled. */
retire: (reason: string, options?: BackgroundToolWakeupRetireOptions) => Promise<boolean>;
+ /** Best effort: makes a waiting delivery claimable now that a result it can
+ * consume exists, e.g. a parent-message projection written without a receipt. */
+ expedite?: () => void;
}
/** Durable ownership repair used by a manual poll after an automatic
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
index ceb95fe540..a30d5a4b49 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
@@ -112,6 +112,25 @@ describe('background tool completion wakeups', () => {
jest.useRealTimers();
});
+ it('expedites its own delivery when a result it can consume appears', async () => {
+ const expedite = jest.fn();
+ const notify = createBackgroundToolCompletionWakeupHandler(
+ async () => ({ deliveryKey: 'delivery-key-1' }),
+ async () => true,
+ async () => true,
+ undefined,
+ expedite,
+ );
+
+ const admission = await notify(registration());
+ if (admission === false) {
+ throw new Error('Expected an admission');
+ }
+ admission.expedite?.();
+
+ expect(expedite).toHaveBeenCalledWith('delivery-key-1');
+ });
+
it('pre-registers the exact task on the invoking response branch', async () => {
const enqueue = jest.fn<
ReturnType<EnqueueBackgroundToolCompletion>,
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts
index 1ca6fca26c..e5ef7ca631 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts
@@ -550,6 +550,7 @@ export function createBackgroundToolCompletionWakeupHandler(
retire: RetireBackgroundToolCompletion,
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
persistResult?: PersistBackgroundToolCompletionResult,
+ expedite?: (deliveryKey: string) => void,
): (
registration: BackgroundToolWakeupRegistration,
) => Promise<BackgroundToolWakeupAdmission | false> {
@@ -613,6 +614,7 @@ export function createBackgroundToolCompletionWakeupHandler(
options == null
? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason)
: retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason, options),
+ ...(expedite == null ? {} : { expedite: () => expedite(admitted.deliveryKey) }),
};
};
}
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index 645a473cf9..37ad6535d6 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -388,6 +388,44 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(JSON.parse(dispatch.content).message).not.toContain('host will resume you');
});
+ it('expedites the delivery when only the parent-message projection was persisted', async () => {
+ const tool = makeSearchTool({ calls: 0 });
+ const expedite = jest.fn();
+ const retire = jest.fn(async () => true);
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({
+ renew: jest.fn(async () => true),
+ persistResult: jest.fn(async () => false),
+ retire,
+ expedite,
+ })),
+ persist: jest.fn(async () => true),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
+ },
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-projection-only',
+ name: tool.name,
+ args: { q: 'projection', run_in_background: true },
+ stepId: 'step-projection-only',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(expedite).toHaveBeenCalledTimes(1);
+ expect(retire).not.toHaveBeenCalled();
+ });
+
it('retires a preregistered delivery when terminal persistence fails', async () => {
const tool = makeSearchTool({ calls: 0 });
const retire = jest.fn(async () => true);
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index f991418273..edbab15caa 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -6130,6 +6130,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
'background tool result was not persisted',
'definite',
);
+ } else if (deliveryReady && !durableReceiptReady) {
+ completionAdmission?.expedite?.();
}
} catch (persistError) {
if (!durableReceiptReady) {
diff --git a/packages/api/src/agents/subagentCompletionWakeup.spec.ts b/packages/api/src/agents/subagentCompletionWakeup.spec.ts
index 16c54710c9..0f09627557 100644
--- a/packages/api/src/agents/subagentCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/subagentCompletionWakeup.spec.ts
@@ -60,7 +60,7 @@ describe('createSubagentCompletionWakeupHandler', () => {
const enqueue = enqueueMock();
const notify = createSubagentCompletionWakeupHandler(enqueue);
- await notify(registration());
+ await expect(notify(registration())).resolves.toBe(true);
expect(enqueue).toHaveBeenCalledTimes(1);
const [envelopeValue, options] = enqueue.mock.calls[0]!;
@@ -112,7 +112,7 @@ describe('createSubagentCompletionWakeupHandler', () => {
const enqueue = enqueueMock();
const notify = createSubagentCompletionWakeupHandler(enqueue);
- await notify(registration({ parentAgentId: undefined }));
+ await expect(notify(registration({ parentAgentId: undefined }))).resolves.toBe(false);
expect(enqueue).not.toHaveBeenCalled();
});
diff --git a/packages/api/src/agents/subagentCompletionWakeup.ts b/packages/api/src/agents/subagentCompletionWakeup.ts
index 93285187bd..0b70b205a2 100644
--- a/packages/api/src/agents/subagentCompletionWakeup.ts
+++ b/packages/api/src/agents/subagentCompletionWakeup.ts
@@ -810,11 +810,11 @@ export function createSubagentCompletionWakeupResolver({
* simply defers until the terminal child message exists. */
export function createSubagentCompletionWakeupHandler(
enqueue: EnqueueAgentTrigger,
-): (registration: SubagentTaskWakeupRegistration) => Promise<void> {
+): (registration: SubagentTaskWakeupRegistration) => Promise<boolean> {
return async (registration) => {
const parentAgentId = registration.parentAgentId?.trim();
if (parentAgentId == null || parentAgentId === '' || isEphemeralAgentId(parentAgentId)) {
- return;
+ return false;
}
const eventId = registration.taskId;
const envelope = createAgentTriggerEnvelope({
@@ -850,5 +850,6 @@ export function createSubagentCompletionWakeupHandler(
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
),
});
+ return true;
};
}
diff --git a/packages/api/src/agents/subagentThreads.spec.ts b/packages/api/src/agents/subagentThreads.spec.ts
index 32daa78c44..410261e711 100644
--- a/packages/api/src/agents/subagentThreads.spec.ts
+++ b/packages/api/src/agents/subagentThreads.spec.ts
@@ -431,6 +431,73 @@ describe('SubagentThreadTaskStore', () => {
});
});
+ it('announces a settled child only after its terminal message is durable', async () => {
+ const userId = 'settled-child-user';
+ const parentConversationId = randomUUID();
+ await saveParent(userId, parentConversationId);
+ const saveMessage = jest.spyOn(methods, 'saveMessage');
+ const terminalSavedAtSettle: boolean[] = [];
+ const store = new SubagentThreadTaskStore(methods, {
+ onTaskPrepared: jest.fn(),
+ onTaskSettled: (settledUserId, settledConversationId, taskIds) => {
+ expect(taskIds).toHaveLength(1);
+ expect(
+ saveMessage.mock.calls.some(
+ ([, message]) =>
+ (message as { messageId?: string }).messageId === `${taskIds[0]}:assistant`,
+ ),
+ ).toBe(true);
+ expect(settledUserId).toBe(userId);
+ expect(settledConversationId).toBe(parentConversationId);
+ terminalSavedAtSettle.push(
+ saveMessage.mock.calls.some(([, message]) =>
+ String((message as { messageId?: string }).messageId).endsWith(':assistant'),
+ ),
+ );
+ },
+ });
+ const config = buildSubagentThreadTaskConfig(
+ store,
+ { userId, parentConversationId },
+ { completionWakeups: true },
+ );
+ const started = config.store.start(
+ taskRequest(config.scopeId, {
+ parentRunId: 'parent-response-1',
+ parentAgentId: 'agent_parent_1',
+ }),
+ );
+ await waitForSettled(store, config.scopeId, started);
+ saveMessage.mockRestore();
+
+ expect(terminalSavedAtSettle).toEqual([true]);
+ });
+
+ it.each([false, true])(
+ 'does not announce a child without an admitted delivery (wakeups enabled: %s)',
+ async (completionWakeups) => {
+ const userId = 'poll-only-settled-user';
+ const parentConversationId = randomUUID();
+ await saveParent(userId, parentConversationId);
+ const onTaskSettled = jest.fn();
+ const store = new SubagentThreadTaskStore(methods, {
+ onTaskPrepared: jest.fn(() => false),
+ onTaskSettled,
+ });
+ const config = buildSubagentThreadTaskConfig(
+ store,
+ { userId, parentConversationId },
+ { completionWakeups },
+ );
+ const started = config.store.start(
+ taskRequest(config.scopeId, { parentRunId: 'parent-response-1' }),
+ );
+ await waitForSettled(store, config.scopeId, started);
+
+ expect(onTaskSettled).not.toHaveBeenCalled();
+ },
+ );
+
it('keeps subagent completion delivery poll-only when wakeups are disabled', async () => {
const userId = 'poll-only-user';
const parentConversationId = randomUUID();
@@ -951,7 +1018,11 @@ describe('SubagentThreadTaskStore', () => {
async (_registration: SubagentTaskWakeupRegistration) => undefined,
);
const firstWorker = new SubagentThreadTaskStore(methods, { onTaskPrepared: firstWakeup });
- const secondWorker = new SubagentThreadTaskStore(methods, { onTaskPrepared: replayWakeup });
+ const onReplaySettled = jest.fn();
+ const secondWorker = new SubagentThreadTaskStore(methods, {
+ onTaskPrepared: replayWakeup,
+ onTaskSettled: onReplaySettled,
+ });
const config = buildSubagentThreadTaskConfig(
firstWorker,
{ userId, parentConversationId },
@@ -999,6 +1070,10 @@ describe('SubagentThreadTaskStore', () => {
expect(firstRun).toHaveBeenCalledTimes(1);
expect(replayRun).not.toHaveBeenCalled();
+ expect(onReplaySettled).toHaveBeenCalledTimes(1);
+ expect(onReplaySettled).toHaveBeenCalledWith(userId, parentConversationId, [
+ requireAccepted(first).task.taskId,
+ ]);
expect(firstWakeup).toHaveBeenCalledTimes(1);
const firstRegistration = firstWakeup.mock.calls[0]?.[0];
const replayRegistration = replayWakeup.mock.calls[0]?.[0];
@@ -1056,8 +1131,16 @@ describe('SubagentThreadTaskStore', () => {
const parentConversationId = randomUUID();
const threadId = randomUUID();
await saveParent(userId, parentConversationId);
- const store = new SubagentThreadTaskStore(methods);
- const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId });
+ const onTaskSettled = jest.fn();
+ const store = new SubagentThreadTaskStore(methods, {
+ onTaskPrepared: jest.fn(),
+ onTaskSettled,
+ });
+ const config = buildSubagentThreadTaskConfig(
+ store,
+ { userId, parentConversationId },
+ { completionWakeups: true },
+ );
await methods.saveConvo(
{ userId },
{
@@ -1095,7 +1178,7 @@ describe('SubagentThreadTaskStore', () => {
},
);
const run = jest.fn(taskRequest(config.scopeId).run);
- const retry = store.start(
+ const retry = config.store.start(
taskRequest(config.scopeId, {
threadId,
idempotencyKey: 'abandoned-attempt',
@@ -1106,6 +1189,11 @@ describe('SubagentThreadTaskStore', () => {
await waitForSettled(store, config.scopeId, retry);
expect(run).not.toHaveBeenCalled();
+ expect(onTaskSettled).toHaveBeenCalledTimes(1);
+ expect(onTaskSettled).toHaveBeenCalledWith(userId, parentConversationId, [
+ requireAccepted(retry).task.taskId,
+ 'abandoned',
+ ]);
expect(store.claim(config.scopeId, requireAccepted(retry).task.taskId)).toMatchObject({
status: 'error',
error:
@@ -1116,6 +1204,32 @@ describe('SubagentThreadTaskStore', () => {
'+subagentTask',
);
expect(messages.map((message) => message.subagentTask?.status)).toEqual(['running', 'error']);
+
+ const replaySettled = jest.fn();
+ const replayWorker = new SubagentThreadTaskStore(methods, {
+ onTaskPrepared: jest.fn(),
+ onTaskSettled: replaySettled,
+ });
+ const replayConfig = buildSubagentThreadTaskConfig(
+ replayWorker,
+ { userId, parentConversationId },
+ { completionWakeups: true },
+ );
+ const replay = replayConfig.store.start(
+ taskRequest(replayConfig.scopeId, {
+ threadId,
+ idempotencyKey: 'abandoned-attempt',
+ requestFingerprint: 'same-inputs',
+ run,
+ }),
+ );
+ await waitForSettled(replayWorker, replayConfig.scopeId, replay);
+ expect(replaySettled).toHaveBeenCalledTimes(1);
+ expect(replaySettled).toHaveBeenCalledWith(userId, parentConversationId, [
+ requireAccepted(retry).task.taskId,
+ 'abandoned',
+ ]);
+ expect(run).not.toHaveBeenCalled();
});
it('holds one active lease per child and exposes provisional ownership safely', async () => {
diff --git a/packages/api/src/agents/subagentThreads.ts b/packages/api/src/agents/subagentThreads.ts
index 355ff40a0d..f3bae71a20 100644
--- a/packages/api/src/agents/subagentThreads.ts
+++ b/packages/api/src/agents/subagentThreads.ts
@@ -140,6 +140,8 @@ interface PreparedThread {
content: string;
taskId: string;
parentRunId: string;
+ /** Original running task closed by a retry under a new terminal message id. */
+ previousTaskId?: string;
};
}
@@ -262,7 +264,14 @@ export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStor
taskId: string;
tenantId?: string;
}) => Promise<boolean>;
- onTaskPrepared?: (registration: SubagentTaskWakeupRegistration) => Promise<void> | void;
+ onTaskPrepared?: (
+ registration: SubagentTaskWakeupRegistration,
+ ) => Promise<boolean | void> | boolean | void;
+ /** Called once a child's terminal message is durable, with the parent conversation its
+ * completion wake-up resumes. The child generation settles before that write, so the
+ * wake-up is ready only from here. Task ids include the original abandoned
+ * attempt when a retry persisted its result under a new id. */
+ onTaskSettled?: (userId: string, parentConversationId: string, taskIds: string[]) => void;
}
export interface SubagentTaskWakeupRegistration {
@@ -616,6 +625,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
private readonly releaseOwnerAdmission?: (userId: string, token: string) => Promise<void>;
private readonly cancelUnroutedTask?: SubagentThreadTaskStoreOptions['cancelUnroutedTask'];
private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared'];
+ private readonly onTaskSettled?: SubagentThreadTaskStoreOptions['onTaskSettled'];
+ /** Tasks whose completion wake-up was registered; only they announce settlement. */
+ private readonly wakeupTaskIds = new Set<string>();
private taskControlTransport?: SubagentTaskControlTransport;
private activityStream = new SubagentActivityStream(new InMemoryEventTransport());
@@ -658,6 +670,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
this.releaseOwnerAdmission = options.releaseOwnerAdmission;
this.cancelUnroutedTask = options.cancelUnroutedTask;
this.onTaskPrepared = options.onTaskPrepared;
+ this.onTaskSettled = options.onTaskSettled;
}
/** Receives payload-free authoritative transitions from the SDK task store. */
@@ -1320,6 +1333,13 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
parentRunId: prepared.replay?.parentRunId ?? request.parentRunId,
createdAt: prepared.taskCreatedAt,
});
+ if (prepared.replay != null) {
+ this.notifyTaskSettled(
+ scope,
+ prepared.replay.taskId,
+ prepared.replay.previousTaskId,
+ );
+ }
if (runtime.signal.aborted) {
throw runtime.signal.reason ?? new Error('Subagent task was cancelled.');
}
@@ -1425,6 +1445,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
}
throw new Error(publicFailureDetail(error));
} finally {
+ if (prepared != null) {
+ this.wakeupTaskIds.delete(prepared.replay?.taskId ?? runtime.taskId);
+ }
if (prepared != null && prepared.replay == null) {
this.completeActivity(
lease,
@@ -2902,6 +2925,10 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
replay: {
status: terminal.subagentTask.status as 'completed' | 'error' | 'cancelled',
taskId: canonicalTaskId,
+ ...(terminal.parentMessageId?.endsWith(':user') === true &&
+ terminal.parentMessageId !== `${canonicalTaskId}:user` && {
+ previousTaskId: terminal.parentMessageId.slice(0, -':user'.length),
+ }),
parentRunId: terminal.subagentTask.parentRunId ?? request.parentRunId,
content:
terminal.text ??
@@ -2956,6 +2983,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
status: 'error',
content: abandonedMessage,
taskId,
+ ...(abandoned.messageId.endsWith(':user') && {
+ previousTaskId: abandoned.messageId.slice(0, -':user'.length),
+ }),
parentRunId: request.parentRunId,
},
};
@@ -3192,7 +3222,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
) {
return;
}
- await this.onTaskPrepared({
+ const admitted = await this.onTaskPrepared({
userId: scope.userId,
parentConversationId: scope.parentConversationId,
parentMessageId: task.parentRunId,
@@ -3203,6 +3233,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
subagentType: request.subagentType,
createdAt: task.createdAt,
});
+ if (admitted !== false) {
+ this.wakeupTaskIds.add(task.taskId);
+ }
}
private async persistCancellation(
@@ -3375,6 +3408,26 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {
}
logger.error(`[subagentThreads] Failed to refresh ${outcome} child thread`, error);
}
+ this.notifyTaskSettled(scope, taskId);
+ }
+
+ private notifyTaskSettled(
+ scope: SubagentThreadScope,
+ taskId: string,
+ previousTaskId?: string,
+ ): void {
+ if (!this.wakeupTaskIds.delete(taskId)) {
+ return;
+ }
+ try {
+ this.onTaskSettled?.(
+ scope.userId,
+ scope.parentConversationId,
+ previousTaskId == null ? [taskId] : [taskId, previousTaskId],
+ );
+ } catch (error) {
+ logger.warn('[subagentThreads] Settled-task listener failed', error);
+ }
}
private aggregateDetachedUsage(detachedUsage: UsageMetadata[]) {
diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md
index 875d797218..5fe25e09aa 100644
--- a/packages/api/src/agents/triggers/README.md
+++ b/packages/api/src/agents/triggers/README.md
@@ -103,7 +103,13 @@ experimental clustered startup:
`completionWaitMaxIntervalMs` caps how long a background or subagent completion waits between
readiness checks while its result or parent turn is not ready. It backs off from 5 seconds by a tenth
-of its age, so a long wait costs about one re-check per cap instead of one every five seconds.
+of its age; a durable result and a settled generation expedite it, so the cap bounds missed signals.
+Parent settlement expedites completion deliveries only in that conversation. A child's durable
+terminal result expedites only its task, plus the original task when recovering an abandoned
+attempt. Terminal replays signal readiness after registering their delivery, without executing the
+child again. A held delivery consumes its wake marker in the same fenced write that releases it
+for ordering or defers readiness. These writes use classic operators for DocumentDB compatibility;
+older workers can ignore the optional marker and fall back to the configured polling interval.
Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data
migration is needed; optional activity reporting preserves the existing numeric/boolean results.
diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts
index c5532179a6..697834d4f8 100644
--- a/packages/api/src/agents/triggers/engine.spec.ts
+++ b/packages/api/src/agents/triggers/engine.spec.ts
@@ -123,6 +123,92 @@ describe('createAgentTriggerDeliveryEngine', () => {
expect(store.retry).not.toHaveBeenCalled();
});
+ it('re-dispatches at once a delivery its deferral found expedited while held', async () => {
+ const dispatchesWithin = async (deferred: boolean | 'expedited'): Promise<number> => {
+ jest.useFakeTimers();
+ try {
+ /** Models the store: the deferred row is claimable again only when the
+ * deferral honored a wake marker and moved it to now. */
+ let claimable = true;
+ const store = storeWith({
+ claimNext: jest.fn(async () => {
+ if (!claimable) {
+ return null;
+ }
+ claimable = false;
+ return delivery();
+ }),
+ defer: jest.fn(async () => {
+ claimable = deferred === 'expedited';
+ return deferred;
+ }),
+ });
+ let dispatches = 0;
+ const dispatch = jest.fn(async () => {
+ dispatches++;
+ if (dispatches > 1) {
+ return successResult();
+ }
+ throw new AgentTriggerExecutionError('The parent generation has not settled yet.', {
+ mode: 'continue',
+ certainty: 'definite',
+ code: 'PARENT_NOT_READY',
+ retryable: true,
+ deferWithoutAttempt: true,
+ retryAfter: '30',
+ });
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START, workerId: 'worker-1' },
+ { concurrency: 1, tickMs: 60_000, maxIdleTickMs: 60_000 },
+ );
+ engine.start();
+ await jest.advanceTimersByTimeAsync(100);
+ await engine.stop();
+ return dispatches;
+ } finally {
+ jest.useRealTimers();
+ }
+ };
+
+ await expect(dispatchesWithin(true)).resolves.toBe(1);
+ await expect(dispatchesWithin('expedited')).resolves.toBe(2);
+ });
+
+ it('reclaims an ordering release made due by a held wake signal without waiting for its stale deadline', async () => {
+ jest.useFakeTimers();
+ try {
+ let claimable = true;
+ const store = storeWith({
+ claimNext: jest.fn(async () => {
+ if (!claimable) return null;
+ claimable = false;
+ return delivery();
+ }),
+ findEarlierUnsettled: jest
+ .fn()
+ .mockResolvedValueOnce({ availableAt: new Date(START.getTime() + 60_000) })
+ .mockResolvedValue(null),
+ release: jest.fn(async () => {
+ claimable = true;
+ return true;
+ }),
+ });
+ const dispatch = jest.fn(async () => successResult());
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START },
+ { concurrency: 1, tickMs: 60_000, maxIdleTickMs: 60_000 },
+ );
+ engine.start();
+ await jest.advanceTimersByTimeAsync(100);
+ await engine.stop();
+ expect(store.release).toHaveBeenCalledTimes(1);
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
it('runs each claim pass under the root trace context', async () => {
const withContext = jest.spyOn(context, 'with');
const store = storeWith({ claimNext: jest.fn(async () => null) });
diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts
index ac5bf350e2..e6d6f9e8fc 100644
--- a/packages/api/src/agents/triggers/engine.ts
+++ b/packages/api/src/agents/triggers/engine.ts
@@ -158,7 +158,7 @@ export interface AgentTriggerDeliveryStore {
claimToken: string;
attempt: number;
availableAt: Date;
- }) => Promise<boolean>;
+ }) => Promise<boolean | 'expedited'>;
complete: (input: {
id: string;
workerId: string;
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index 1130b7f605..0b1278141a 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -209,6 +209,143 @@ describe('durable agent trigger service', () => {
await service.stop();
});
+ describe('waiting completion deliveries', () => {
+ const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };
+ const flush = () => new Promise((resolve) => setImmediate(resolve));
+ const result = { status: 'completed' as const, output: 'done', settledAt: START };
+
+ it('expedites and claims the delivery whose background result just became durable', async () => {
+ const methods = deliveryMethods({
+ persistAgentBackgroundToolResult: jest.fn(async () => true),
+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),
+ });
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({ address });
+ jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear();
+
+ await expect(
+ service.persistBackgroundToolResult({
+ deliveryKey: 'trigger_background',
+ sourceId: 'background-tool-completion',
+ result,
+ }),
+ ).resolves.toBe(true);
+ await flush();
+
+ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({
+ deliveryKeys: ['trigger_background'],
+ sourceIds: ['background-tool-completion', 'subagent-completion'],
+ now: expect.any(Date),
+ });
+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
+ await service.stop();
+ });
+
+ it('targets only named subagent completions when a child settles', async () => {
+ const methods = deliveryMethods({
+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),
+ });
+ const service = createAgentTriggerService({ methods });
+ await service.initialize({ address });
+ service.expediteCompletionWakeups({
+ user: 'user-1',
+ conversationId: 'parent-1',
+ taskIds: ['child-1'],
+ });
+ await flush();
+ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({
+ user: 'user-1',
+ conversationId: 'parent-1',
+ taskIds: ['child-1'],
+ sourceIds: ['subagent-completion'],
+ now: expect.any(Date),
+ });
+ await service.stop();
+ });
+
+ it('does not expedite a result the store refused to persist', async () => {
+ const methods = deliveryMethods({
+ persistAgentBackgroundToolResult: jest.fn(async () => false),
+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),
+ });
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({ address });
+
+ await expect(
+ service.persistBackgroundToolResult({
+ deliveryKey: 'trigger_background',
+ sourceId: 'background-tool-completion',
+ result,
+ }),
+ ).resolves.toBe(false);
+ await flush();
+
+ expect(methods.expediteAgentTriggerDeliveries).not.toHaveBeenCalled();
+ await service.stop();
+ });
+
+ it("expedites a principal's completion deliveries when one of its generations settles", async () => {
+ const methods = deliveryMethods({
+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })),
+ });
+ let settled: ((event: { userId: string; conversationId: string }) => void) | undefined;
+ const unsubscribe = jest.fn();
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ subscribeGenerationSettled: (listener) => {
+ settled = listener;
+ return unsubscribe;
+ },
+ });
+ await service.initialize({ address });
+ jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear();
+
+ settled?.({ userId: '507f1f77bcf86cd799439011', conversationId: 'conversation-1' });
+ await flush();
+
+ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({
+ user: '507f1f77bcf86cd799439011',
+ conversationId: 'conversation-1',
+ sourceIds: ['background-tool-completion', 'subagent-completion'],
+ now: expect.any(Date),
+ });
+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
+ await service.stop();
+ expect(unsubscribe).toHaveBeenCalledTimes(1);
+ });
+
+ it('runs a claim pass even when nothing moved, since a match may already be due', async () => {
+ const methods = deliveryMethods({
+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 0, held: 0 })),
+ });
+ let settled: ((event: { userId: string; conversationId: string }) => void) | undefined;
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ subscribeGenerationSettled: (listener) => {
+ settled = listener;
+ return () => undefined;
+ },
+ });
+ await service.initialize({ address });
+ jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear();
+
+ settled?.({ userId: '507f1f77bcf86cd799439011', conversationId: 'conversation-1' });
+ await flush();
+
+ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledTimes(1);
+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
+ await service.stop();
+ });
+ });
+
describe('completion wait configuration', () => {
const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index f208336e57..fc2125bef3 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -26,7 +26,9 @@ import type {
import type { AgentTriggerEnqueueOptions, PreparedAgentTriggerDelivery } from './delivery';
import type { BoundAddress } from '../../app/origin';
import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine';
+import { BACKGROUND_TOOL_COMPLETION_SOURCE } from '../backgroundCompletionWakeup';
import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown';
+import { SUBAGENT_COMPLETION_SOURCE } from '../subagentCompletionWakeup';
import { generateAgentTriggerToken } from '../../crypto/jwt';
import { prepareAgentTriggerDelivery } from './delivery';
import { selfOriginFromAddress } from '../../app/origin';
@@ -35,6 +37,9 @@ import { parseAgentTriggerEnvelope } from './envelope';
import { createIdleRecoveryLoop } from '../recovery';
import { WAITING_RETRY_CAP_MS } from './backoff';
+/** Internal sources whose deliveries wait on a result or on their parent generation. */
+const COMPLETION_WAKEUP_SOURCES = [BACKGROUND_TOOL_COMPLETION_SOURCE, SUBAGENT_COMPLETION_SOURCE];
+
export const AGENT_TRIGGER_TOKEN_TTL = '60s';
const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;
const DEFAULT_USER_DRAIN_POLL_MS = 100;
@@ -53,6 +58,11 @@ export interface AgentTriggerServiceOptions {
};
}
+/** A principal's deliveries resuming one conversation, or exact deliveries. */
+export type AgentTriggerCompletionExpedite =
+ | { user: string; conversationId: string; taskIds?: string[] }
+ | { deliveryKeys: string[] };
+
export interface AgentTriggerServiceDeps {
fetch?: AgentTriggerExecutionHostDeps['fetch'];
getTimezone?: AgentTriggerExecutionHostDeps['getTimezone'];
@@ -69,6 +79,16 @@ export interface AgentTriggerServiceDeps {
reclaimCheckpointDeletions?: (limit: number, activity?: { found: boolean }) => Promise<number>;
supportsDetachedActionCompletion?: () => boolean;
settleSourceBeforeDeadLetter?: AgentTriggerDeliveryEngineDeps['settleSourceBeforeDeadLetter'];
+ /** Subscribes to generations reaching a terminal state; returns an unsubscribe. */
+ subscribeGenerationSettled?: (
+ listener: (event: AgentTriggerGenerationSettledEvent) => void,
+ ) => () => void;
+}
+
+/** The part of a settled generation that decides which waiting deliveries it may unblock. */
+export interface AgentTriggerGenerationSettledEvent {
+ userId: string;
+ conversationId: string;
}
export interface AgentTriggerDeliveryReceipt {
@@ -124,6 +144,7 @@ export interface AgentTriggerDeliveryPersistence {
retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];
renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];
persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
+ expediteAgentTriggerDeliveries?: AgentTriggerDeliveryMethods['expediteAgentTriggerDeliveries'];
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
@@ -210,6 +231,8 @@ export interface AgentTriggerService {
getBackgroundCompletionResultBatchSize: () => number;
/** Longest a waiting completion delivery re-checks readiness. */
getCompletionWaitMaxIntervalMs: () => number;
+ /** Best effort: moves waiting completion deliveries forward after what they wait on changed. */
+ expediteCompletionWakeups: (input: AgentTriggerCompletionExpedite) => void;
releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
drainUser: (userId: string) => Promise<void>;
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
@@ -539,9 +562,45 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
void purgeRecoveryLoop?.start();
};
+ /** Moves waiting completion deliveries forward when what they wait on has
+ * changed, and marks held ones so their next deferral re-checks at once. The
+ * claim pass always runs: a matching delivery may already be due here without
+ * having moved. Best effort: a missed expedite only means the delivery
+ * re-checks at its backoff instead of immediately. */
+ const expediteCompletions = (input: AgentTriggerCompletionExpedite): void => {
+ const expedite = deps.methods?.expediteAgentTriggerDeliveries;
+ if (expedite == null || !deliveryReady || stopping) {
+ return;
+ }
+ void runAsSystem(() =>
+ expedite({
+ ...('user' in input
+ ? {
+ user: input.user,
+ conversationId: input.conversationId,
+ ...(input.taskIds != null && { taskIds: input.taskIds }),
+ }
+ : { deliveryKeys: input.deliveryKeys }),
+ sourceIds:
+ 'user' in input && input.taskIds != null
+ ? [SUBAGENT_COMPLETION_SOURCE]
+ : COMPLETION_WAKEUP_SOURCES,
+ now: new Date(),
+ }),
+ )
+ .then(() => deliveryEngine?.wake())
+ .catch((error) =>
+ logger.warn('[agent-triggers] failed to expedite waiting completion deliveries:', error),
+ );
+ };
+
+ let unsubscribeGenerationSettled: (() => void) | undefined;
+
const stop = async (): Promise<void> => {
stopping = true;
deliveryReady = false;
+ unsubscribeGenerationSettled?.();
+ unsubscribeGenerationSettled = undefined;
await purgeRecoveryLoop?.stop();
await initializePromise?.catch(() => undefined);
await deliveryEngine?.stop();
@@ -611,6 +670,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
);
deliveryReady = true;
deliveryEngine.start();
+ /** Deliveries waiting on a parent resume that parent's conversation, so a
+ * settled generation wakes only those, not every waiting task of the user. */
+ unsubscribeGenerationSettled ??= deps.subscribeGenerationSettled?.(
+ ({ userId, conversationId }) => expediteCompletions({ user: userId, conversationId }),
+ );
startPurgeRecovery();
logger.info('[agent-triggers] durable delivery engine started');
}).finally(() => {
@@ -727,7 +791,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
persistBackgroundToolResult: (input) =>
runAsSystem(async () => {
const persist = requireMethods().persistAgentBackgroundToolResult;
- return persist == null ? false : persist(input);
+ const persisted = persist == null ? false : await persist(input);
+ if (persisted) {
+ expediteCompletions({ deliveryKeys: [input.deliveryKey] });
+ }
+ return persisted;
}),
getBackgroundToolResultClaim: (input) =>
runAsSystem(async () => {
@@ -736,6 +804,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}),
getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,
getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,
+ expediteCompletionWakeups: (input) => expediteCompletions(input),
releaseBackgroundToolResultClaims: (input) =>
runAsSystem(async () => {
const release = requireMethods().releaseAgentBackgroundToolResultClaims;
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 991ee1b882..a92318e29f 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -568,10 +568,22 @@ export interface CreateGenerationJobOptions {
* receiving this claim, then pass the same object to {@link finishTerminalJob}
* from a `finally` block.
*/
+/** A generation reached a terminal state and released its runtime. */
+export interface GenerationSettledEvent {
+ streamId: string;
+ conversationId: string;
+ userId: string;
+ status: TerminalJobClaim['status'];
+}
+
+export type GenerationSettledListener = (event: GenerationSettledEvent) => void;
+
export interface TerminalJobClaim {
readonly streamId: string;
readonly createdAt: number;
readonly conversationId?: string;
+ /** The generation's owner, so settlement can be announced to that principal's waiters. */
+ readonly userId?: string;
readonly status: 'complete' | 'error' | 'aborted';
readonly error?: string;
/** The winner must durably publish either its normal FINAL or a
@@ -786,6 +798,7 @@ class GenerationJobManagerClass {
/** Makes terminal cleanup idempotent while keeping claims opaque to callers. */
private terminalFinishPromises = new WeakMap<TerminalJobClaim, Promise<void>>();
+ private generationSettledListeners = new Set<GenerationSettledListener>();
/** Exact local runtime observed when a claim won; never clean a later runtime. */
private terminalClaimRuntimes = new WeakMap<TerminalJobClaim, RuntimeJobState | null>();
@@ -1078,6 +1091,7 @@ class GenerationJobManagerClass {
createdAt,
...(job?.createdAt === createdAt &&
job.conversationId != null && { conversationId: job.conversationId }),
+ ...(job?.createdAt === createdAt && job.userId != null && { userId: job.userId }),
status: 'error' as const,
error,
drainedSteers: Object.freeze([...drainedSteers]),
@@ -2256,7 +2270,7 @@ class GenerationJobManagerClass {
streamId: string,
job: Pick<
SerializableJobData,
- 'createdAt' | 'conversationId' | 'providerExecutionId' | 'agentEventDeliveryKey'
+ 'createdAt' | 'conversationId' | 'providerExecutionId' | 'agentEventDeliveryKey' | 'userId'
>,
message: string,
): Promise<boolean> {
@@ -2288,6 +2302,13 @@ class GenerationJobManagerClass {
expectCreatedAt: job.createdAt,
})
) {
+ /** A direct terminal transition builds no claim, so it announces itself. */
+ this.notifyGenerationSettled({
+ streamId,
+ conversationId: job.conversationId,
+ userId: job.userId,
+ status: 'error',
+ });
return true;
}
} catch (error) {
@@ -2298,6 +2319,15 @@ class GenerationJobManagerClass {
try {
const current = await this.jobStore.getJob(streamId);
+ if (current?.createdAt === job.createdAt && current.status === 'error') {
+ this.notifyGenerationSettled({
+ streamId,
+ conversationId: current.conversationId,
+ userId: current.userId,
+ status: 'error',
+ });
+ return true;
+ }
if (
current == null ||
current.createdAt !== job.createdAt ||
@@ -3901,6 +3931,7 @@ class GenerationJobManagerClass {
...(jobData.conversationId != null && {
conversationId: jobData.conversationId,
}),
+ ...(jobData.userId != null && { userId: jobData.userId }),
status,
...(terminalError != null && { error: terminalError }),
...(options.persistencePending === true && {
@@ -4071,6 +4102,40 @@ class GenerationJobManagerClass {
* claim is idempotent, and every local mutation is pinned to the runtime
* object and generation epoch captured when the CAS won.
*/
+ /**
+ * Calls `listener` after each generation owned by this process reaches a
+ * terminal state and its runtime is released — by completion, error, or
+ * abort. Listeners run synchronously and must not throw; failures are logged
+ * and never affect terminal cleanup. Returns an unsubscribe function.
+ */
+ onGenerationSettled(listener: GenerationSettledListener): () => void {
+ this.generationSettledListeners.add(listener);
+ return () => {
+ this.generationSettledListeners.delete(listener);
+ };
+ }
+
+ private notifyGenerationSettled(
+ target: Pick<TerminalJobClaim, 'streamId' | 'conversationId' | 'userId' | 'status'>,
+ ): void {
+ if (target.userId == null || this.generationSettledListeners.size === 0) {
+ return;
+ }
+ const event: GenerationSettledEvent = {
+ streamId: target.streamId,
+ conversationId: target.conversationId ?? target.streamId,
+ userId: target.userId,
+ status: target.status,
+ };
+ for (const listener of this.generationSettledListeners) {
+ try {
+ listener(event);
+ } catch (listenerError) {
+ logger.error('[GenerationJobManager] Generation settled listener failed', listenerError);
+ }
+ }
+ }
+
finishTerminalJob(claim: TerminalJobClaim): Promise<void> {
const inFlight = this.terminalFinishPromises.get(claim);
if (inFlight) {
@@ -4260,6 +4325,7 @@ class GenerationJobManagerClass {
metricStatus = 'error';
}
recordGenerationJob(this.storeLabel, metricStatus);
+ this.notifyGenerationSettled(claim);
}
if (cleanupError != null) {
@@ -4616,6 +4682,7 @@ class GenerationJobManagerClass {
...(jobData.conversationId != null && {
conversationId: jobData.conversationId,
}),
+ ...(jobData.userId != null && { userId: jobData.userId }),
status: 'aborted',
persistencePending: true,
drainedSteers: Object.freeze([...drainedSteers]),
@@ -8865,6 +8932,17 @@ class GenerationJobManagerClass {
await this.runApprovalExpiredHandler(streamId, expiredJob);
await this.notifyApprovalExpiredRuntime(streamId, expiredJob.createdAt, observedRuntime);
+ /** Expiry is a direct `requires_action -> aborted` transition that never builds a
+ * terminal claim, so it announces settlement itself. */
+ this.notifyGenerationSettled({
+ streamId,
+ conversationId: expiredJob.conversationId,
+ userId: expiredJob.userId,
+ status: 'aborted',
+ });
+ /** Terminal now; releasing ownership keeps the sweep's relay branch from
+ * announcing this generation a second time. */
+ this.releaseJobOwnership(streamId, expiredJob.createdAt);
return true;
}
@@ -9055,7 +9133,17 @@ class GenerationJobManagerClass {
await this.runApprovalExpiredHandler(streamId, job);
}
await this.notifyApprovalExpiredRuntime(streamId, job.createdAt, runtime);
- changed = this.releaseJobOwnership(streamId, job.createdAt) || changed;
+ const released = this.releaseJobOwnership(streamId, job.createdAt);
+ if (released) {
+ /** The store won the expiry CAS, so no local claim announced it. */
+ this.notifyGenerationSettled({
+ streamId,
+ conversationId: job.conversationId,
+ userId: job.userId,
+ status: 'aborted',
+ });
+ }
+ changed = released || changed;
continue;
}
if (
diff --git a/packages/api/src/stream/__tests__/generationSettled.spec.ts b/packages/api/src/stream/__tests__/generationSettled.spec.ts
new file mode 100644
index 0000000000..473b548dce
--- /dev/null
+++ b/packages/api/src/stream/__tests__/generationSettled.spec.ts
@@ -0,0 +1,145 @@
+/**
+ * Waiting completion deliveries are expedited when a generation settles, so the
+ * settled notification must fire exactly once for every terminal path and name
+ * the generation's owner — and a failing listener must never disturb cleanup.
+ */
+import type { GenerationSettledEvent } from '../GenerationJobManager';
+import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
+
+/** Suppress winston Console transport output (survives jest.resetModules) */
+jest.spyOn(console, 'log').mockImplementation();
+
+async function configureManager() {
+ const { GenerationJobManager } = await import('../GenerationJobManager');
+ const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');
+ const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');
+
+ GenerationJobManager.configure({
+ jobStore: new InMemoryJobStore(),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ GenerationJobManager.initialize();
+ return GenerationJobManager;
+}
+
+describe('generation settled notifications', () => {
+ beforeEach(() => {
+ jest.resetModules();
+ });
+
+ it('announces a completed generation once, with its owner', async () => {
+ const manager = await configureManager();
+ const events: GenerationSettledEvent[] = [];
+ manager.onGenerationSettled((event) => events.push(event));
+ const job = await manager.createJob('settled-complete', 'user-1', 'conversation-1');
+
+ await manager.completeJob('settled-complete', undefined, job.createdAt);
+ await manager.completeJob('settled-complete', undefined, job.createdAt);
+
+ expect(events).toEqual([
+ {
+ streamId: 'settled-complete',
+ conversationId: 'conversation-1',
+ userId: 'user-1',
+ status: 'complete',
+ },
+ ]);
+ await manager.destroy();
+ });
+
+ it('announces a failed generation', async () => {
+ const manager = await configureManager();
+ const events: GenerationSettledEvent[] = [];
+ manager.onGenerationSettled((event) => events.push(event));
+ const job = await manager.createJob('settled-error', 'user-2');
+
+ await manager.completeJob('settled-error', 'provider failed', job.createdAt);
+
+ expect(events).toEqual([
+ expect.objectContaining({ streamId: 'settled-error', userId: 'user-2', status: 'error' }),
+ ]);
+ await manager.destroy();
+ });
+
+ it('announces an aborted generation', async () => {
+ const manager = await configureManager();
+ const events: GenerationSettledEvent[] = [];
+ manager.onGenerationSettled((event) => events.push(event));
+ await manager.createJob('settled-abort', 'user-3');
+
+ const result = await manager.abortJob('settled-abort');
+
+ expect(result.success).toBe(true);
+ expect(events).toEqual([
+ expect.objectContaining({ streamId: 'settled-abort', userId: 'user-3', status: 'aborted' }),
+ ]);
+ await manager.destroy();
+ });
+
+ it('announces a generation whose approval expired', async () => {
+ const manager = await configureManager();
+ const events: GenerationSettledEvent[] = [];
+ manager.onGenerationSettled((event) => events.push(event));
+ await manager.createJob('settled-approval', 'user-6', 'conversation-6');
+ const action = buildPendingAction(
+ buildToolApprovalPayload([
+ { name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_expire' },
+ ]),
+ {
+ streamId: 'settled-approval',
+ conversationId: 'conversation-6',
+ runId: 'run-1',
+ responseMessageId: 'msg-1',
+ },
+ );
+ await manager.approvals.pause('settled-approval', action);
+ expect(events).toEqual([]);
+
+ await expect(manager.expireApproval('settled-approval')).resolves.toBe(true);
+ await expect(manager.expireApproval('settled-approval')).resolves.toBe(false);
+ /** The next expiry sweep relays the terminal job without announcing it again. */
+ await (manager as unknown as { cleanup(): Promise<void> }).cleanup();
+
+ expect(events).toEqual([
+ {
+ streamId: 'settled-approval',
+ conversationId: 'conversation-6',
+ userId: 'user-6',
+ status: 'aborted',
+ },
+ ]);
+ await manager.destroy();
+ });
+
+ it('keeps cleanup and later listeners running when one listener throws', async () => {
+ const manager = await configureManager();
+ const events: GenerationSettledEvent[] = [];
+ manager.onGenerationSettled(() => {
+ throw new Error('listener failed');
+ });
+ manager.onGenerationSettled((event) => events.push(event));
+ const job = await manager.createJob('settled-throwing-listener', 'user-4');
+
+ await expect(
+ manager.completeJob('settled-throwing-listener', undefined, job.createdAt),
+ ).resolves.toBe(true);
+
+ expect(events).toHaveLength(1);
+ await manager.destroy();
+ });
+
+ it('stops notifying after unsubscribe', async () => {
+ const manager = await configureManager();
+ const listener = jest.fn();
+ const unsubscribe = manager.onGenerationSettled(listener);
+ unsubscribe();
+ const job = await manager.createJob('settled-unsubscribed', 'user-5');
+
+ await manager.completeJob('settled-unsubscribed', undefined, job.createdAt);
+
+ expect(listener).not.toHaveBeenCalled();
+ await manager.destroy();
+ });
+});
diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
index 3b5d964d9d..1c8578d86d 100644
--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
@@ -1070,48 +1070,86 @@ describe('GenerationJobManager start-generation claim', () => {
expect(Object.getOwnPropertyDescriptor(durable!, 'replacedJobs')).toBeUndefined();
});
- it('terminalizes the exact committed epoch when legacy verification fails during recovery', async () => {
- const streamId = 'stream-lost-create-legacy-failure';
- const clientRequestId = 'req-lost-create-legacy-failure';
- const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
- const actualCreate = store.createJob.bind(store);
- jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {
- await actualCreate(...args);
- throw new Error('simulated lost atomic create reply');
- });
- const actualClaim = store.claimIdempotencyKey.bind(store);
- let failLegacyProbe = true;
- jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => {
- if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) {
- failLegacyProbe = false;
- return Promise.reject(new Error('simulated legacy probe outage'));
- }
- return actualClaim(key, value, ttlSeconds);
- });
+ it.each([
+ { loseTerminalReply: false, replaced: false },
+ { loseTerminalReply: true, replaced: false },
+ { loseTerminalReply: true, replaced: true },
+ ])(
+ 'announces only the exact recovered epoch (lost reply: $loseTerminalReply, replaced: $replaced)',
+ async ({ loseTerminalReply, replaced }) => {
+ const settled = jest.fn();
+ manager.onGenerationSettled(settled);
+ const transition = store.transitionStatus.bind(store);
+ jest.spyOn(store, 'transitionStatus').mockImplementation(async (id, input) => {
+ const committed = await transition(id, input);
+ if (loseTerminalReply && input.to === 'error' && committed) {
+ if (replaced) {
+ const previous = await store.getJob(id);
+ const replacement = await actualCreate(id, 'user-1', id);
+ await transition(id, {
+ from: 'running',
+ to: 'error',
+ expectCreatedAt: replacement.createdAt,
+ patch: { error: previous?.error, finalEvent: previous?.finalEvent },
+ });
+ }
+ throw new Error('lost terminal CAS reply');
+ }
+ return committed;
+ });
+ const streamId = 'stream-lost-create-legacy-failure';
+ const clientRequestId = 'req-lost-create-legacy-failure';
+ const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ const actualCreate = store.createJob.bind(store);
+ jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {
+ await actualCreate(...args);
+ throw new Error('simulated lost atomic create reply');
+ });
+ const actualClaim = store.claimIdempotencyKey.bind(store);
+ let failLegacyProbe = true;
+ jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => {
+ if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) {
+ failLegacyProbe = false;
+ return Promise.reject(new Error('simulated legacy probe outage'));
+ }
+ return actualClaim(key, value, ttlSeconds);
+ });
- await expect(
- manager.createJob(streamId, 'user-1', streamId, {
- idempotencyClientRequestId: clientRequestId,
- idempotencyClaimToken: claim.existing!.claimToken,
- initialMetadata: { generationProtocolVersion: 2 },
- }),
- ).rejects.toThrow('simulated lost atomic create reply');
+ await expect(
+ manager.createJob(streamId, 'user-1', streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('simulated lost atomic create reply');
- const durable = await store.getJob(streamId);
- expect(durable).toMatchObject({
- status: 'error',
- error: 'Generation idempotency rollout fence could not be recovered',
- finalEvent: expect.stringContaining('terminal_payload_missing'),
- });
- const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
- expect(retry).toMatchObject({
- claimed: false,
- existing: { startedAt: durable!.createdAt },
- });
- await expect(
- manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),
- ).resolves.toMatchObject({ claimed: false });
- });
+ const durable = await store.getJob(streamId);
+ expect(durable).toMatchObject({
+ status: 'error',
+ error: 'Generation idempotency rollout fence could not be recovered',
+ finalEvent: expect.stringContaining('terminal_payload_missing'),
+ });
+ if (replaced) {
+ expect(settled).not.toHaveBeenCalled();
+ return;
+ }
+ expect(settled).toHaveBeenCalledTimes(1);
+ expect(settled).toHaveBeenCalledWith({
+ streamId,
+ conversationId: streamId,
+ userId: 'user-1',
+ status: 'error',
+ });
+ const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: durable!.createdAt },
+ });
+ await expect(
+ manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ },
+ );
it('terminalizes and preserves the primary fence when the legacy started mark fails', async () => {
const claim = await manager.claimGeneration(
diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts
index b6c97e6098..ef72b19ee5 100644
--- a/packages/api/src/stream/index.ts
+++ b/packages/api/src/stream/index.ts
@@ -4,6 +4,8 @@ export {
type CreateGenerationJobOptions,
type GenerationJobManagerOptions,
type TerminalJobClaim,
+ type GenerationSettledEvent,
+ type GenerationSettledListener,
GENERATION_RECOVERY_FAILED_ERROR,
TERMINAL_PUBLICATION_RECONNECT_ERROR,
} from './GenerationJobManager';
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index 3c6850d7b3..7498b932c4 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -453,6 +453,348 @@ describe('agent trigger delivery methods', () => {
).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil });
});
+ describe('expediteAgentTriggerDeliveries', () => {
+ const background = { id: 'background-tool-completion', type: 'internal' };
+ const later = new Date(START.getTime() + 30_000);
+ const capable = {
+ workerId: 'background-capable-worker',
+ claimToken: 'background-capable-claim',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
+ };
+ const waiting = (overrides: Partial<Parameters<typeof enqueueInput>[0]> = {}) =>
+ methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ orderingKey: `background-lane-${counter + 1}`,
+ envelope: { event: { source: background } },
+ requiredWorkerCapability:
+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
+ availableAt: later,
+ ...overrides,
+ }),
+ );
+
+ it('makes a deferred delivery claimable now once its result is durable', async () => {
+ const target = await waiting();
+ const sibling = await waiting();
+ await expect(methods.claimNextAgentTriggerDelivery(capable)).resolves.toBeNull();
+
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ deliveryKeys: [target.delivery.deliveryKey],
+ sourceIds: [background.id],
+ now: START,
+ }),
+ ).resolves.toEqual({ expedited: 1, held: 0 });
+
+ await expect(methods.claimNextAgentTriggerDelivery(capable)).resolves.toMatchObject({
+ id: target.delivery.id,
+ });
+ await expect(
+ methods.claimNextAgentTriggerDelivery({ ...capable, claimToken: 'second-claim' }),
+ ).resolves.toBeNull();
+ const untouched = await Delivery.findById(sibling.delivery.id).lean();
+ expect(untouched?.availableAt).toEqual(later);
+ });
+
+ it("moves only the principal's waiting rows from the named sources", async () => {
+ const user = new mongoose.Types.ObjectId();
+ const mine = await waiting({ user });
+ const otherUser = await waiting();
+ const otherSource = await waiting({
+ user,
+ envelope: { event: { source: { id: 'agent-queued-turn', type: 'internal' } } },
+ });
+ const external = await waiting({
+ user,
+ envelope: { event: { source: { id: background.id, type: 'webhook' } } },
+ });
+ const due = await waiting({ user, availableAt: START });
+
+ await expect(
+ methods.expediteAgentTriggerDeliveries({ user, sourceIds: [background.id], now: START }),
+ ).resolves.toEqual({ expedited: 1, held: 0 });
+
+ const rows = await Delivery.find({
+ _id: {
+ $in: [mine, otherUser, otherSource, external, due].map((row) => row.delivery.id),
+ },
+ }).lean();
+ const availableAt = new Map(rows.map((row) => [String(row._id), row.availableAt]));
+ expect(availableAt.get(mine.delivery.id)).toEqual(START);
+ expect(availableAt.get(otherUser.delivery.id)).toEqual(later);
+ expect(availableAt.get(otherSource.delivery.id)).toEqual(later);
+ expect(availableAt.get(external.delivery.id)).toEqual(later);
+ expect(availableAt.get(due.delivery.id)).toEqual(START);
+ });
+
+ it('marks a delivery a worker currently holds instead of moving it', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const held = await waiting({ user, availableAt: START });
+ const claim = await methods.claimNextAgentTriggerDelivery(capable);
+ expect(claim).toMatchObject({ id: held.delivery.id });
+ const before = await Delivery.findById(held.delivery.id).lean();
+
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ user,
+ sourceIds: [background.id],
+ now: new Date(START.getTime() - 60_000),
+ }),
+ ).resolves.toEqual({ expedited: 0, held: 1 });
+
+ const after = await Delivery.findById(held.delivery.id).lean();
+ expect(after?.status).toBe(before?.status);
+ expect(after?.availableAt).toEqual(before?.availableAt);
+ expect(after?.wakeRequestedAt).toEqual(new Date(START.getTime() - 60_000));
+ });
+
+ it('re-checks at once when readiness changed while the delivery was held', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const held = await waiting({ user, availableAt: START });
+ const claim = await methods.claimNextAgentTriggerDelivery(capable);
+ expect(claim).toMatchObject({ id: held.delivery.id });
+ const fence = {
+ id: held.delivery.id,
+ workerId: capable.workerId,
+ claimToken: capable.claimToken,
+ };
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...fence, now: START });
+ await expect(
+ methods.expediteAgentTriggerDeliveries({ user, sourceIds: [background.id], now: START }),
+ ).resolves.toEqual({ expedited: 0, held: 1 });
+
+ const beforeDefer = Date.now();
+ await expect(
+ methods.deferAgentTriggerDeliveryAttempt({
+ ...fence,
+ attempt: attempt!,
+ availableAt: later,
+ }),
+ ).resolves.toBe('expedited');
+
+ const deferred = await Delivery.findById(held.delivery.id).lean();
+ expect(deferred?.wakeRequestedAt).toBeUndefined();
+ expect(deferred?.availableAt.getTime()).toBeGreaterThanOrEqual(beforeDefer);
+ expect(deferred?.availableAt).not.toEqual(later);
+
+ const reclaimed = await methods.claimNextAgentTriggerDelivery({
+ ...capable,
+ claimToken: 'second-claim',
+ now: new Date(),
+ leaseUntil: new Date(Date.now() + 60_000),
+ });
+ expect(reclaimed).toMatchObject({ id: held.delivery.id });
+ const secondFence = { ...fence, claimToken: 'second-claim' };
+ const secondAttempt = await methods.beginAgentTriggerDeliveryAttempt({
+ ...secondFence,
+ now: new Date(),
+ });
+ await expect(
+ methods.deferAgentTriggerDeliveryAttempt({
+ ...secondFence,
+ attempt: secondAttempt!,
+ availableAt: later,
+ }),
+ ).resolves.toBe(true);
+ expect((await Delivery.findById(held.delivery.id).lean())?.availableAt).toEqual(later);
+ });
+
+ it.each(['ordinary', 'legacy', 'shielded'] as const)(
+ 'honors a held wake marker on ordering release for %s leases',
+ async (profile) => {
+ const row = await waiting({
+ availableAt: START,
+ ...(profile === 'ordinary' && { requiredWorkerCapability: undefined }),
+ });
+ if (profile === 'legacy') {
+ await Delivery.updateOne(
+ { _id: row.delivery.id },
+ {
+ $set: { status: 'capability_pending', availableAt: START },
+ $unset: { capabilityStatus: 1, leaseUntil: 1 },
+ },
+ );
+ }
+ const claimed = await methods.claimNextAgentTriggerDelivery(capable);
+ expect(claimed?.id).toBe(row.delivery.id);
+ const fence = {
+ id: row.delivery.id,
+ workerId: capable.workerId,
+ claimToken: capable.claimToken,
+ };
+ await methods.expediteAgentTriggerDeliveries({
+ deliveryKeys: [row.delivery.deliveryKey],
+ sourceIds: [background.id],
+ now: START,
+ });
+ await expect(
+ methods.releaseAgentTriggerDelivery({
+ ...fence,
+ claimToken: 'stale',
+ availableAt: later,
+ }),
+ ).resolves.toBe(false);
+ expect((await Delivery.findById(row.delivery.id).lean())?.wakeRequestedAt).toEqual(START);
+ await expect(
+ methods.releaseAgentTriggerDelivery({ ...fence, availableAt: later }),
+ ).resolves.toBe(true);
+ const released = await Delivery.findById(row.delivery.id).lean();
+ expect(released?.wakeRequestedAt).toBeUndefined();
+ expect(released?.availableAt.getTime()).toBeLessThanOrEqual(Date.now());
+ expect(released?.claimAvailableAt).toEqual(released?.availableAt);
+ expect(released?.attempts).toBe(0);
+ const reclaimed = await methods.claimNextAgentTriggerDelivery({
+ ...capable,
+ now: new Date(),
+ claimToken: 'next',
+ });
+ expect(reclaimed?.id).toBe(row.delivery.id);
+ await methods.releaseAgentTriggerDelivery({
+ ...fence,
+ claimToken: 'next',
+ availableAt: later,
+ });
+ expect((await Delivery.findById(row.delivery.id).lean())?.availableAt).toEqual(later);
+ },
+ );
+
+ it.each(['release', 'defer'] as const)(
+ 'does not lose an expedite between the unmarked and marked %s writes',
+ async (operation) => {
+ const row = await waiting({ availableAt: START });
+ await methods.claimNextAgentTriggerDelivery(capable);
+ const fence = {
+ id: row.delivery.id,
+ workerId: capable.workerId,
+ claimToken: capable.claimToken,
+ };
+ const attempt =
+ operation === 'defer'
+ ? await methods.beginAgentTriggerDeliveryAttempt({ ...fence, now: START })
+ : undefined;
+ const updateOne = Delivery.collection.updateOne.bind(Delivery.collection);
+ let injected = false;
+ const spy = jest
+ .spyOn(Delivery.collection, 'updateOne')
+ .mockImplementation(async (filter, update, options) => {
+ if (
+ !injected &&
+ filter.capabilityClaimToken === capable.claimToken &&
+ (filter.wakeRequestedAt as { $exists?: boolean } | undefined)?.$exists === false
+ ) {
+ injected = true;
+ await methods.expediteAgentTriggerDeliveries({
+ deliveryKeys: [row.delivery.deliveryKey],
+ sourceIds: [background.id],
+ now: START,
+ });
+ }
+ return updateOne(filter, update, options);
+ });
+ try {
+ const input = { ...fence, availableAt: later };
+ if (operation === 'defer') {
+ await expect(
+ methods.deferAgentTriggerDeliveryAttempt({ ...input, attempt: attempt! }),
+ ).resolves.toBe('expedited');
+ } else {
+ await expect(methods.releaseAgentTriggerDelivery(input)).resolves.toBe(true);
+ }
+ } finally {
+ spy.mockRestore();
+ }
+ expect(injected).toBe(true);
+ const released = await Delivery.findById(row.delivery.id).lean();
+ expect(released?.wakeRequestedAt).toBeUndefined();
+ expect(released?.availableAt.getTime()).toBeLessThanOrEqual(Date.now());
+ expect(released?.attempts).toBe(0);
+ },
+ );
+
+ it('leaves unfinished siblings backed off when only named child tasks settle', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const source = { id: 'subagent-completion', type: 'internal' };
+ const task = (taskId: string) =>
+ waiting({
+ user,
+ envelope: {
+ event: { source, payload: { taskId } },
+ target: { conversationId: 'parent' },
+ },
+ });
+ const original = await task('original');
+ const recovered = await task('recovered');
+ const sibling = await task('sibling');
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ user,
+ conversationId: 'parent',
+ taskIds: ['original', 'recovered'],
+ sourceIds: [source.id],
+ now: START,
+ }),
+ ).resolves.toEqual({ expedited: 2, held: 0 });
+ expect((await Delivery.findById(original.delivery.id).lean())?.availableAt).toEqual(START);
+ expect((await Delivery.findById(recovered.delivery.id).lean())?.availableAt).toEqual(START);
+ expect((await Delivery.findById(sibling.delivery.id).lean())?.availableAt).toEqual(later);
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ user,
+ conversationId: 'parent',
+ taskIds: [],
+ sourceIds: [source.id],
+ now: START,
+ }),
+ ).rejects.toThrow(TypeError);
+ });
+
+ it("narrows a principal's selection to the conversation being resumed", async () => {
+ const user = new mongoose.Types.ObjectId();
+ const target = await waiting({
+ user,
+ envelope: { event: { source: background }, target: { conversationId: 'settled-convo' } },
+ });
+ const other = await waiting({
+ user,
+ envelope: { event: { source: background }, target: { conversationId: 'other-convo' } },
+ });
+
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ user,
+ conversationId: 'settled-convo',
+ sourceIds: [background.id],
+ now: START,
+ }),
+ ).resolves.toEqual({ expedited: 1, held: 0 });
+
+ expect((await Delivery.findById(target.delivery.id).lean())?.availableAt).toEqual(START);
+ expect((await Delivery.findById(other.delivery.id).lean())?.availableAt).toEqual(later);
+ });
+
+ it('refuses an unbounded or malformed selection', async () => {
+ await expect(
+ methods.expediteAgentTriggerDeliveries({ sourceIds: [background.id], now: START }),
+ ).rejects.toThrow(TypeError);
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ user: new mongoose.Types.ObjectId(),
+ sourceIds: [],
+ now: START,
+ }),
+ ).rejects.toThrow(TypeError);
+ await expect(
+ methods.expediteAgentTriggerDeliveries({
+ deliveryKeys: ['trigger_x'],
+ sourceIds: [background.id],
+ now: new Date(Number.NaN),
+ }),
+ ).rejects.toThrow(TypeError);
+ });
+ });
+
it('persists one private background result receipt independently of message rows', async () => {
const source = { id: 'background-tool-completion', type: 'internal' };
const queued = await methods.enqueueAgentTriggerDelivery(
@@ -2468,7 +2810,7 @@ describe('agent trigger delivery methods', () => {
);
});
- it("indexes a user's waiting deliveries for user-scoped readiness reads", async () => {
+ it("indexes a user's waiting deliveries for settle-time expediting", async () => {
const deliveryIndexes = await Delivery.collection.indexes();
const userIndex = deliveryIndexes.find(
(index) =>
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
index e3fe4c2d0f..1e2b6ead6f 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -246,6 +246,27 @@ export interface AgentEventActorReceiptStorageMetrics {
deadDeliveries: number;
}
+/** Selects deferred internal deliveries whose readiness condition just changed. */
+export interface ExpediteAgentTriggerDeliveriesInput {
+ sourceIds: readonly string[];
+ /** Exact deliveries, e.g. the one whose result just became durable. */
+ deliveryKeys?: readonly string[];
+ /** Every matching delivery of one principal, e.g. after one of its generations settled. */
+ user?: string | Types.ObjectId;
+ /** Narrows a principal's selection to deliveries that resume this conversation. */
+ conversationId?: string;
+ /** Exact tasks within a principal's conversation, including a repaired attempt's predecessor. */
+ taskIds?: readonly string[];
+ now: Date;
+}
+
+export interface ExpediteAgentTriggerDeliveriesResult {
+ /** Deferred, unheld deliveries moved to `now`. */
+ expedited: number;
+ /** Deliveries a worker held, marked so their next deferral re-checks at once. */
+ held: number;
+}
+
export interface AgentTriggerDeliveryMethods {
ensureAgentTriggerDeliveryIndexes: () => Promise<void>;
enqueueAgentTriggerDelivery: (
@@ -270,9 +291,10 @@ export interface AgentTriggerDeliveryMethods {
beginAgentTriggerDeliveryAttempt: (
input: AgentTriggerDeliveryFence & { now: Date },
) => Promise<number | null>;
+ /** `expedited` when readiness changed while the delivery was held: it is due now instead. */
deferAgentTriggerDeliveryAttempt: (
input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },
- ) => Promise<boolean>;
+ ) => Promise<boolean | 'expedited'>;
completeAgentTriggerDelivery: (
input: AgentTriggerDeliveryFence & {
attempt: number;
@@ -307,6 +329,9 @@ export interface AgentTriggerDeliveryMethods {
sourceId: string;
now: Date;
}) => Promise<AgentTriggerProducerLeaseStatus>;
+ expediteAgentTriggerDeliveries: (
+ input: ExpediteAgentTriggerDeliveriesInput,
+ ) => Promise<ExpediteAgentTriggerDeliveriesResult>;
persistAgentBackgroundToolResult: (
input: PersistAgentBackgroundToolResultInput,
) => Promise<boolean>;
@@ -1846,57 +1871,61 @@ export function createAgentTriggerDeliveryMethods(
$or: [ordinaryFence(input), legacyCapabilityFence(input), shieldCapabilityFence(input)],
});
+ /** Readiness signals and the lease release must meet in one fenced write. Try
+ * the unmarked state first, then the marked state: a concurrent expedite can
+ * only add a marker while this lease is held. After release, expedite's second
+ * update sees an unheld row. No pipeline updates or unfenced follow-up needed. */
+ async function releaseWaitingDelivery(
+ input: AgentTriggerDeliveryFence & { availableAt: Date; attempt?: number },
+ ): Promise<boolean | 'expedited'> {
+ const attemptFence = input.attempt == null ? {} : { attempts: input.attempt };
+ const attemptChange = input.attempt == null ? {} : { $inc: { attempts: -1 } };
+ const profiles = [
+ { filter: shieldCapabilityFence(input), status: 'leased', capabilityStatus: 'pending' },
+ { filter: legacyCapabilityFence(input), status: 'capability_pending' },
+ { filter: ordinaryFence(input), status: 'pending' },
+ ] as const;
+ for (const marked of [false, true]) {
+ for (const profile of profiles) {
+ const availableAt = marked ? new Date() : input.availableAt;
+ const result = await Delivery().updateOne(
+ {
+ _id: input.id,
+ ...profile.filter,
+ ...attemptFence,
+ wakeRequestedAt: { $exists: marked },
+ },
+ {
+ ...attemptChange,
+ $set: {
+ status: profile.status,
+ ...('capabilityStatus' in profile && { capabilityStatus: profile.capabilityStatus }),
+ availableAt,
+ claimAvailableAt: availableAt,
+ },
+ $unset: {
+ leaseBy: 1,
+ leaseUntil: 1,
+ claimToken: 1,
+ capabilityLeaseBy: 1,
+ capabilityLeaseUntil: 1,
+ capabilityClaimToken: 1,
+ wakeRequestedAt: 1,
+ },
+ },
+ );
+ if (result.modifiedCount === 1) {
+ return marked ? 'expedited' : true;
+ }
+ }
+ }
+ return false;
+ }
+
async function releaseAgentTriggerDelivery(
input: AgentTriggerDeliveryFence & { availableAt: Date },
): Promise<boolean> {
- const shieldResult = await Delivery().updateOne(
- { _id: input.id, ...shieldCapabilityFence(input) },
- {
- $set: {
- status: 'leased',
- availableAt: input.availableAt,
- capabilityStatus: 'pending',
- claimAvailableAt: input.availableAt,
- },
- $unset: {
- leaseBy: 1,
- leaseUntil: 1,
- claimToken: 1,
- capabilityLeaseBy: 1,
- capabilityLeaseUntil: 1,
- capabilityClaimToken: 1,
- },
- },
- );
- if (shieldResult.modifiedCount === 1) {
- return true;
- }
- const capabilityResult = await Delivery().updateOne(
- { _id: input.id, ...legacyCapabilityFence(input) },
- {
- $set: {
- status: 'capability_pending',
- availableAt: input.availableAt,
- claimAvailableAt: input.availableAt,
- },
- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
- },
- );
- if (capabilityResult.modifiedCount === 1) {
- return true;
- }
- const result = await Delivery().updateOne(
- { _id: input.id, ...ordinaryFence(input) },
- {
- $set: {
- status: 'pending',
- availableAt: input.availableAt,
- claimAvailableAt: input.availableAt,
- },
- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
- },
- );
- return result.modifiedCount === 1;
+ return (await releaseWaitingDelivery(input)) !== false;
}
async function beginAgentTriggerDeliveryAttempt(
@@ -1926,64 +1955,11 @@ export function createAgentTriggerDeliveryMethods(
/** Releases a pre-dispatch deferral and restores the attempt consumed by beginAttempt. */
async function deferAgentTriggerDeliveryAttempt(
input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },
- ): Promise<boolean> {
+ ): Promise<boolean | 'expedited'> {
if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) {
throw new TypeError('attempt must be a positive integer');
}
- const update = {
- $inc: { attempts: -1 },
- $set: {
- availableAt: input.availableAt,
- claimAvailableAt: input.availableAt,
- },
- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
- };
- const shieldResult = await Delivery().updateOne(
- {
- _id: input.id,
- ...shieldCapabilityFence(input),
- attempts: input.attempt,
- },
- {
- $inc: update.$inc,
- $set: {
- status: 'leased',
- availableAt: input.availableAt,
- capabilityStatus: 'pending',
- claimAvailableAt: input.availableAt,
- },
- $unset: {
- leaseBy: 1,
- leaseUntil: 1,
- claimToken: 1,
- capabilityLeaseBy: 1,
- capabilityLeaseUntil: 1,
- capabilityClaimToken: 1,
- },
- },
- );
- if (shieldResult.modifiedCount === 1) {
- return true;
- }
- const capabilityResult = await Delivery().updateOne(
- {
- _id: input.id,
- ...legacyCapabilityFence(input),
- attempts: input.attempt,
- },
- {
- ...update,
- $set: { ...update.$set, status: 'capability_pending' },
- },
- );
- if (capabilityResult.modifiedCount === 1) {
- return true;
- }
- const result = await Delivery().updateOne(
- { _id: input.id, ...ordinaryFence(input), attempts: input.attempt },
- { ...update, $set: { ...update.$set, status: 'pending' } },
- );
- return result.modifiedCount === 1;
+ return releaseWaitingDelivery(input);
}
async function completeAgentTriggerDelivery(
@@ -2287,6 +2263,75 @@ export function createAgentTriggerDeliveryMethods(
: { status: 'expired', leaseUntil: delivery.producerLeaseUntil };
}
+ /** Pulls deferred deliveries back to `now` when the condition they were
+ * waiting on has changed, so a waiting delivery can back off without delaying
+ * the moment it becomes deliverable. Unclaimed rows move now; held rows retain
+ * a signal consumed atomically by readiness deferral or ordering release. */
+ async function expediteAgentTriggerDeliveries(
+ input: ExpediteAgentTriggerDeliveriesInput,
+ ): Promise<ExpediteAgentTriggerDeliveriesResult> {
+ const deliveryKeys = input.deliveryKeys ?? [];
+ if (
+ (input.taskIds != null &&
+ (input.user == null ||
+ input.conversationId == null ||
+ input.taskIds.length === 0 ||
+ input.taskIds.some((id) => id.length === 0 || id.length > 256))) ||
+ input.sourceIds.length === 0 ||
+ input.sourceIds.some((id) => id.length === 0 || id.length > 256) ||
+ deliveryKeys.some((key) => key.length === 0 || key.length > 256) ||
+ (deliveryKeys.length === 0 && input.user == null) ||
+ (input.conversationId != null &&
+ (input.conversationId.length === 0 || input.conversationId.length > 256)) ||
+ !(input.now instanceof Date) ||
+ !Number.isFinite(input.now.getTime())
+ ) {
+ throw new TypeError('Invalid agent trigger delivery expedite');
+ }
+ const selection = {
+ 'envelope.event.source.type': 'internal',
+ 'envelope.event.source.id': { $in: [...input.sourceIds] },
+ ...(deliveryKeys.length > 0 && { deliveryKey: { $in: [...deliveryKeys] } }),
+ ...(input.taskIds != null && {
+ 'envelope.event.payload.taskId': { $in: [...input.taskIds] },
+ }),
+ ...(input.user != null && { user: input.user }),
+ ...(input.conversationId != null && {
+ 'envelope.target.conversationId': input.conversationId,
+ }),
+ };
+ /** Classic operators only: aggregation-pipeline updates are not portable. A
+ * held row may be deferred on readiness its worker read before the change,
+ * so it keeps a marker that its deferral honors instead of moving now. The
+ * marker is written first: a row released after it was read as held is then
+ * seen unheld by the move, so no release between the two escapes both. */
+ const held = await Delivery().updateMany(
+ {
+ ...selection,
+ status: { $in: ['leased', 'capability_leased'] },
+ $or: [{ leaseBy: { $exists: true } }, { capabilityLeaseBy: { $exists: true } }],
+ },
+ { $set: { wakeRequestedAt: input.now } },
+ );
+ const moved = await Delivery().updateMany(
+ {
+ ...selection,
+ availableAt: { $gt: input.now },
+ leaseBy: { $exists: false },
+ $or: [
+ { status: { $in: ['pending', 'capability_pending'] } },
+ {
+ status: 'leased',
+ capabilityStatus: 'pending',
+ capabilityLeaseBy: { $exists: false },
+ },
+ ],
+ },
+ { $set: { availableAt: input.now, claimAvailableAt: input.now } },
+ );
+ return { expedited: moved.modifiedCount, held: held.matchedCount };
+ }
+
/** Stores terminal output on the pre-admitted delivery before attempting the
* parent-message projection. The first terminal receipt wins; exact retries
* are idempotent and conflicting rewrites fail closed. */
@@ -4049,6 +4094,7 @@ export function createAgentTriggerDeliveryMethods(
retireAgentTriggerDelivery,
renewAgentTriggerDeliveryProducerLease,
getAgentTriggerDeliveryProducerLease,
+ expediteAgentTriggerDeliveries,
persistAgentBackgroundToolResult,
getAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
index 2d08f6d14c..d9b503e6c5 100644
--- a/packages/data-schemas/src/schema/triggerDelivery.ts
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -218,6 +218,7 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
requeueCount: { type: Number, default: 0, min: 0 },
stagingRecoveryAt: { type: Date },
laneCleanupPendingAt: { type: Date },
+ wakeRequestedAt: { type: Date },
},
{ timestamps: true },
);
@@ -253,7 +254,7 @@ triggerDeliverySchema.index(
{ sparse: true },
);
triggerDeliverySchema.index({ status: 1, updatedAt: -1 });
-/** One user's waiting deliveries, read when their readiness changes. */
+/** One user's waiting deliveries, read when that user's generation settles. */
triggerDeliverySchema.index({ user: 1, status: 1, availableAt: 1 });
triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });
triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });
diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts
index be019d65a2..5865d88267 100644
--- a/packages/data-schemas/src/types/triggerDelivery.ts
+++ b/packages/data-schemas/src/types/triggerDelivery.ts
@@ -171,6 +171,8 @@ export interface IAgentTriggerDelivery {
stagingRecoveryAt?: Date;
/** Durable proof that successful settlement still owes lane cleanup publication. */
laneCleanupPendingAt?: Date;
+ /** Readiness changed while a worker held this delivery; its next deferral re-checks at once. */
+ wakeRequestedAt?: Date;
createdAt?: Date;
updatedAt?: Date;
}