Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
⚡ fix: Deliver Waiting Completion Wake-ups the Moment They Are Ready (#16339)
Danny Avila7238be3e5ea1
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.jsindex 329bfd4635..f8944c6ea8 100644--- a/api/server/services/Agents/triggers.js+++ b/api/server/services/Agents/triggers.js@@ -53,6 +53,7 @@ service = createAgentTriggerService({),isPrincipalActive: methods.isAgentTriggerPrincipalActive,supportsDetachedActionCompletion: () => GenerationJobManager.supportsDetachedAgentEventActions,+ subscribeGenerationSettled: (listener) => GenerationJobManager.onGenerationSettled(listener),settleSourceBeforeDeadLetter: queuedTurnLifecycle.settleBeforeDeadLetter,prepareContinue: createAgentContinuationResolver({eventActor: eventActorAdapter,@@ -92,6 +93,7 @@ module.exports = {retireAgentTrigger: service.retire,renewAgentTriggerProducerLease: service.renewProducerLease,persistAgentBackgroundToolResult: service.persistBackgroundToolResult,+ expediteCompletionWakeups: service.expediteCompletionWakeups,getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,drainAgentTriggerDeliveriesForUser: service.drainUser,diff --git a/api/server/services/Endpoints/agents/backgroundCompletion.js b/api/server/services/Endpoints/agents/backgroundCompletion.jsindex 8ca1cae6ef..da93d98242 100644--- a/api/server/services/Endpoints/agents/backgroundCompletion.js+++ b/api/server/services/Endpoints/agents/backgroundCompletion.js@@ -11,6 +11,7 @@ const {releaseAgentBackgroundToolResultClaims,renewAgentTriggerProducerLease,retireAgentTrigger,+ expediteCompletionWakeups,} = require('../../Agents/triggers');const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeupHandler(@@ -19,6 +20,7 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeuprenewAgentTriggerProducerLease,(deliveryKey, sourceId, result) =>persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),+ (deliveryKey) => expediteCompletionWakeups({ deliveryKeys: [deliveryKey] }),);function createBackgroundToolResultPersistence({ req, updateToolCallResult }) {diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.js b/api/server/services/Endpoints/agents/subagentThreadStore.jsindex b33b7df541..d98f0f275f 100644--- a/api/server/services/Endpoints/agents/subagentThreadStore.js+++ b/api/server/services/Endpoints/agents/subagentThreadStore.js@@ -12,7 +12,7 @@ const {SubagentActivityStream,} = require('@librechat/api');const db = require('~/models');-const { enqueueAgentTrigger } = require('../../Agents/triggers');+const { enqueueAgentTrigger, expediteCompletionWakeups } = require('../../Agents/triggers');const GENERATION_DRAIN_TIMEOUT_MS = 45_000;const GENERATION_DRAIN_POLL_MS = 100;@@ -78,6 +78,8 @@ const subagentThreadTaskStore = createSubagentThreadTaskStore(releaseOwnerAdmission: db.releaseSubagentAdmission,cancelUnroutedTask: cancelUnroutedGeneration,onTaskPrepared: completionWakeupHandler,+ onTaskSettled: (userId, conversationId, taskIds) =>+ expediteCompletionWakeups({ user: userId, conversationId, taskIds }),},);diff --git a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js b/api/server/services/Endpoints/agents/subagentThreadStore.spec.jsindex bae8a2cdeb..a7d2609468 100644--- a/api/server/services/Endpoints/agents/subagentThreadStore.spec.js+++ b/api/server/services/Endpoints/agents/subagentThreadStore.spec.js@@ -45,6 +45,7 @@ jest.mock('~/models', () => ({jest.mock('../../Agents/triggers', () => ({enqueueAgentTrigger: jest.fn(),+ expediteCompletionWakeups: jest.fn(),}));const {@@ -80,6 +81,16 @@ describe('subagent thread Redis lifecycle', () => {expect(mockCompletionWakeupHandler).toHaveBeenCalledWith({ taskId: 'task-1' });});+ it('expedites only the settled task identities in their parent conversation', () => {+ const { expediteCompletionWakeups } = require('../../Agents/triggers');+ taskStoreOptions.onTaskSettled('user-1', 'parent-1', ['task-1', 'recovered-task']);+ expect(expediteCompletionWakeups).toHaveBeenCalledWith({+ user: 'user-1',+ conversationId: 'parent-1',+ taskIds: ['task-1', 'recovered-task'],+ });+ });+it('registers local task-store quiescence independently of optional Redis setup', () => {expect(taskStoreShutdownRegistration).toEqual(['subagent task store',diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.tsindex 8ef1d74cca..02d57cb7d8 100644--- a/packages/api/src/agents/backgroundCompletion.ts+++ b/packages/api/src/agents/backgroundCompletion.ts@@ -42,6 +42,9 @@ export interface BackgroundToolWakeupAdmission {* Manual polling requires an atomic unclaimed-only transition: once a* resolver owns a lease, its prepared continuation cannot be cancelled. */retire: (reason: string, options?: BackgroundToolWakeupRetireOptions) => Promise<boolean>;+ /** Best effort: makes a waiting delivery claimable now that a result it can+ * consume exists, e.g. a parent-message projection written without a receipt. */+ expedite?: () => void;}/** Durable ownership repair used by a manual poll after an automaticdiff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.tsindex ceb95fe540..a30d5a4b49 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts@@ -112,6 +112,25 @@ describe('background tool completion wakeups', () => {jest.useRealTimers();});+ it('expedites its own delivery when a result it can consume appears', async () => {+ const expedite = jest.fn();+ const notify = createBackgroundToolCompletionWakeupHandler(+ async () => ({ deliveryKey: 'delivery-key-1' }),+ async () => true,+ async () => true,+ undefined,+ expedite,+ );++ const admission = await notify(registration());+ if (admission === false) {+ throw new Error('Expected an admission');+ }+ admission.expedite?.();++ expect(expedite).toHaveBeenCalledWith('delivery-key-1');+ });+it('pre-registers the exact task on the invoking response branch', async () => {const enqueue = jest.fn<ReturnType<EnqueueBackgroundToolCompletion>,diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.tsindex 1ca6fca26c..e5ef7ca631 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts@@ -550,6 +550,7 @@ export function createBackgroundToolCompletionWakeupHandler(retire: RetireBackgroundToolCompletion,renewProducerLease: RenewBackgroundToolCompletionProducerLease,persistResult?: PersistBackgroundToolCompletionResult,+ expedite?: (deliveryKey: string) => void,): (registration: BackgroundToolWakeupRegistration,) => Promise<BackgroundToolWakeupAdmission | false> {@@ -613,6 +614,7 @@ export function createBackgroundToolCompletionWakeupHandler(options == null? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason): retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason, options),+ ...(expedite == null ? {} : { expedite: () => expedite(admitted.deliveryKey) }),};};}diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.tsindex 645a473cf9..37ad6535d6 100644--- a/packages/api/src/agents/handlers.background.spec.ts+++ b/packages/api/src/agents/handlers.background.spec.ts@@ -388,6 +388,44 @@ describe('createToolExecuteHandler — background tool calls', () => {expect(JSON.parse(dispatch.content).message).not.toContain('host will resume you');});+ it('expedites the delivery when only the parent-message projection was persisted', async () => {+ const tool = makeSearchTool({ calls: 0 });+ const expedite = jest.fn();+ const retire = jest.fn(async () => true);+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: {+ preregister: jest.fn(async () => ({+ renew: jest.fn(async () => true),+ persistResult: jest.fn(async () => false),+ retire,+ expedite,+ })),+ persist: jest.fn(async () => true),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),+ },+ });++ await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-projection-only',+ name: tool.name,+ args: { q: 'projection', run_in_background: true },+ stepId: 'step-projection-only',+ },+ ],+ agentId: 'agent_parent_1',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'exec_convo', run_id: 'response-1' },+ });+ await flushMicrotasks();+ await flushMicrotasks();++ expect(expedite).toHaveBeenCalledTimes(1);+ expect(retire).not.toHaveBeenCalled();+ });+it('retires a preregistered delivery when terminal persistence fails', async () => {const tool = makeSearchTool({ calls: 0 });const retire = jest.fn(async () => true);diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.tsindex f991418273..edbab15caa 100644--- a/packages/api/src/agents/handlers.ts+++ b/packages/api/src/agents/handlers.ts@@ -6130,6 +6130,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand'background tool result was not persisted','definite',);+ } else if (deliveryReady && !durableReceiptReady) {+ completionAdmission?.expedite?.();}} catch (persistError) {if (!durableReceiptReady) {diff --git a/packages/api/src/agents/subagentCompletionWakeup.spec.ts b/packages/api/src/agents/subagentCompletionWakeup.spec.tsindex 16c54710c9..0f09627557 100644--- a/packages/api/src/agents/subagentCompletionWakeup.spec.ts+++ b/packages/api/src/agents/subagentCompletionWakeup.spec.ts@@ -60,7 +60,7 @@ describe('createSubagentCompletionWakeupHandler', () => {const enqueue = enqueueMock();const notify = createSubagentCompletionWakeupHandler(enqueue);- await notify(registration());+ await expect(notify(registration())).resolves.toBe(true);expect(enqueue).toHaveBeenCalledTimes(1);const [envelopeValue, options] = enqueue.mock.calls[0]!;@@ -112,7 +112,7 @@ describe('createSubagentCompletionWakeupHandler', () => {const enqueue = enqueueMock();const notify = createSubagentCompletionWakeupHandler(enqueue);- await notify(registration({ parentAgentId: undefined }));+ await expect(notify(registration({ parentAgentId: undefined }))).resolves.toBe(false);expect(enqueue).not.toHaveBeenCalled();});diff --git a/packages/api/src/agents/subagentCompletionWakeup.ts b/packages/api/src/agents/subagentCompletionWakeup.tsindex 93285187bd..0b70b205a2 100644--- a/packages/api/src/agents/subagentCompletionWakeup.ts+++ b/packages/api/src/agents/subagentCompletionWakeup.ts@@ -810,11 +810,11 @@ export function createSubagentCompletionWakeupResolver({* simply defers until the terminal child message exists. */export function createSubagentCompletionWakeupHandler(enqueue: EnqueueAgentTrigger,-): (registration: SubagentTaskWakeupRegistration) => Promise<void> {+): (registration: SubagentTaskWakeupRegistration) => Promise<boolean> {return async (registration) => {const parentAgentId = registration.parentAgentId?.trim();if (parentAgentId == null || parentAgentId === '' || isEphemeralAgentId(parentAgentId)) {- return;+ return false;}const eventId = registration.taskId;const envelope = createAgentTriggerEnvelope({@@ -850,5 +850,6 @@ export function createSubagentCompletionWakeupHandler(Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,),});+ return true;};}diff --git a/packages/api/src/agents/subagentThreads.spec.ts b/packages/api/src/agents/subagentThreads.spec.tsindex 32daa78c44..410261e711 100644--- a/packages/api/src/agents/subagentThreads.spec.ts+++ b/packages/api/src/agents/subagentThreads.spec.ts@@ -431,6 +431,73 @@ describe('SubagentThreadTaskStore', () => {});});+ it('announces a settled child only after its terminal message is durable', async () => {+ const userId = 'settled-child-user';+ const parentConversationId = randomUUID();+ await saveParent(userId, parentConversationId);+ const saveMessage = jest.spyOn(methods, 'saveMessage');+ const terminalSavedAtSettle: boolean[] = [];+ const store = new SubagentThreadTaskStore(methods, {+ onTaskPrepared: jest.fn(),+ onTaskSettled: (settledUserId, settledConversationId, taskIds) => {+ expect(taskIds).toHaveLength(1);+ expect(+ saveMessage.mock.calls.some(+ ([, message]) =>+ (message as { messageId?: string }).messageId === `${taskIds[0]}:assistant`,+ ),+ ).toBe(true);+ expect(settledUserId).toBe(userId);+ expect(settledConversationId).toBe(parentConversationId);+ terminalSavedAtSettle.push(+ saveMessage.mock.calls.some(([, message]) =>+ String((message as { messageId?: string }).messageId).endsWith(':assistant'),+ ),+ );+ },+ });+ const config = buildSubagentThreadTaskConfig(+ store,+ { userId, parentConversationId },+ { completionWakeups: true },+ );+ const started = config.store.start(+ taskRequest(config.scopeId, {+ parentRunId: 'parent-response-1',+ parentAgentId: 'agent_parent_1',+ }),+ );+ await waitForSettled(store, config.scopeId, started);+ saveMessage.mockRestore();++ expect(terminalSavedAtSettle).toEqual([true]);+ });++ it.each([false, true])(+ 'does not announce a child without an admitted delivery (wakeups enabled: %s)',+ async (completionWakeups) => {+ const userId = 'poll-only-settled-user';+ const parentConversationId = randomUUID();+ await saveParent(userId, parentConversationId);+ const onTaskSettled = jest.fn();+ const store = new SubagentThreadTaskStore(methods, {+ onTaskPrepared: jest.fn(() => false),+ onTaskSettled,+ });+ const config = buildSubagentThreadTaskConfig(+ store,+ { userId, parentConversationId },+ { completionWakeups },+ );+ const started = config.store.start(+ taskRequest(config.scopeId, { parentRunId: 'parent-response-1' }),+ );+ await waitForSettled(store, config.scopeId, started);++ expect(onTaskSettled).not.toHaveBeenCalled();+ },+ );+it('keeps subagent completion delivery poll-only when wakeups are disabled', async () => {const userId = 'poll-only-user';const parentConversationId = randomUUID();@@ -951,7 +1018,11 @@ describe('SubagentThreadTaskStore', () => {async (_registration: SubagentTaskWakeupRegistration) => undefined,);const firstWorker = new SubagentThreadTaskStore(methods, { onTaskPrepared: firstWakeup });- const secondWorker = new SubagentThreadTaskStore(methods, { onTaskPrepared: replayWakeup });+ const onReplaySettled = jest.fn();+ const secondWorker = new SubagentThreadTaskStore(methods, {+ onTaskPrepared: replayWakeup,+ onTaskSettled: onReplaySettled,+ });const config = buildSubagentThreadTaskConfig(firstWorker,{ userId, parentConversationId },@@ -999,6 +1070,10 @@ describe('SubagentThreadTaskStore', () => {expect(firstRun).toHaveBeenCalledTimes(1);expect(replayRun).not.toHaveBeenCalled();+ expect(onReplaySettled).toHaveBeenCalledTimes(1);+ expect(onReplaySettled).toHaveBeenCalledWith(userId, parentConversationId, [+ requireAccepted(first).task.taskId,+ ]);expect(firstWakeup).toHaveBeenCalledTimes(1);const firstRegistration = firstWakeup.mock.calls[0]?.[0];const replayRegistration = replayWakeup.mock.calls[0]?.[0];@@ -1056,8 +1131,16 @@ describe('SubagentThreadTaskStore', () => {const parentConversationId = randomUUID();const threadId = randomUUID();await saveParent(userId, parentConversationId);- const store = new SubagentThreadTaskStore(methods);- const config = buildSubagentThreadTaskConfig(store, { userId, parentConversationId });+ const onTaskSettled = jest.fn();+ const store = new SubagentThreadTaskStore(methods, {+ onTaskPrepared: jest.fn(),+ onTaskSettled,+ });+ const config = buildSubagentThreadTaskConfig(+ store,+ { userId, parentConversationId },+ { completionWakeups: true },+ );await methods.saveConvo({ userId },{@@ -1095,7 +1178,7 @@ describe('SubagentThreadTaskStore', () => {},);const run = jest.fn(taskRequest(config.scopeId).run);- const retry = store.start(+ const retry = config.store.start(taskRequest(config.scopeId, {threadId,idempotencyKey: 'abandoned-attempt',@@ -1106,6 +1189,11 @@ describe('SubagentThreadTaskStore', () => {await waitForSettled(store, config.scopeId, retry);expect(run).not.toHaveBeenCalled();+ expect(onTaskSettled).toHaveBeenCalledTimes(1);+ expect(onTaskSettled).toHaveBeenCalledWith(userId, parentConversationId, [+ requireAccepted(retry).task.taskId,+ 'abandoned',+ ]);expect(store.claim(config.scopeId, requireAccepted(retry).task.taskId)).toMatchObject({status: 'error',error:@@ -1116,6 +1204,32 @@ describe('SubagentThreadTaskStore', () => {'+subagentTask',);expect(messages.map((message) => message.subagentTask?.status)).toEqual(['running', 'error']);++ const replaySettled = jest.fn();+ const replayWorker = new SubagentThreadTaskStore(methods, {+ onTaskPrepared: jest.fn(),+ onTaskSettled: replaySettled,+ });+ const replayConfig = buildSubagentThreadTaskConfig(+ replayWorker,+ { userId, parentConversationId },+ { completionWakeups: true },+ );+ const replay = replayConfig.store.start(+ taskRequest(replayConfig.scopeId, {+ threadId,+ idempotencyKey: 'abandoned-attempt',+ requestFingerprint: 'same-inputs',+ run,+ }),+ );+ await waitForSettled(replayWorker, replayConfig.scopeId, replay);+ expect(replaySettled).toHaveBeenCalledTimes(1);+ expect(replaySettled).toHaveBeenCalledWith(userId, parentConversationId, [+ requireAccepted(retry).task.taskId,+ 'abandoned',+ ]);+ expect(run).not.toHaveBeenCalled();});it('holds one active lease per child and exposes provisional ownership safely', async () => {diff --git a/packages/api/src/agents/subagentThreads.ts b/packages/api/src/agents/subagentThreads.tsindex 355ff40a0d..f3bae71a20 100644--- a/packages/api/src/agents/subagentThreads.ts+++ b/packages/api/src/agents/subagentThreads.ts@@ -140,6 +140,8 @@ interface PreparedThread {content: string;taskId: string;parentRunId: string;+ /** Original running task closed by a retry under a new terminal message id. */+ previousTaskId?: string;};}@@ -262,7 +264,14 @@ export interface SubagentThreadTaskStoreOptions extends InMemorySubagentTaskStortaskId: string;tenantId?: string;}) => Promise<boolean>;- onTaskPrepared?: (registration: SubagentTaskWakeupRegistration) => Promise<void> | void;+ onTaskPrepared?: (+ registration: SubagentTaskWakeupRegistration,+ ) => Promise<boolean | void> | boolean | void;+ /** Called once a child's terminal message is durable, with the parent conversation its+ * completion wake-up resumes. The child generation settles before that write, so the+ * wake-up is ready only from here. Task ids include the original abandoned+ * attempt when a retry persisted its result under a new id. */+ onTaskSettled?: (userId: string, parentConversationId: string, taskIds: string[]) => void;}export interface SubagentTaskWakeupRegistration {@@ -616,6 +625,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {private readonly releaseOwnerAdmission?: (userId: string, token: string) => Promise<void>;private readonly cancelUnroutedTask?: SubagentThreadTaskStoreOptions['cancelUnroutedTask'];private readonly onTaskPrepared?: SubagentThreadTaskStoreOptions['onTaskPrepared'];+ private readonly onTaskSettled?: SubagentThreadTaskStoreOptions['onTaskSettled'];+ /** Tasks whose completion wake-up was registered; only they announce settlement. */+ private readonly wakeupTaskIds = new Set<string>();private taskControlTransport?: SubagentTaskControlTransport;private activityStream = new SubagentActivityStream(new InMemoryEventTransport());@@ -658,6 +670,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {this.releaseOwnerAdmission = options.releaseOwnerAdmission;this.cancelUnroutedTask = options.cancelUnroutedTask;this.onTaskPrepared = options.onTaskPrepared;+ this.onTaskSettled = options.onTaskSettled;}/** Receives payload-free authoritative transitions from the SDK task store. */@@ -1320,6 +1333,13 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {parentRunId: prepared.replay?.parentRunId ?? request.parentRunId,createdAt: prepared.taskCreatedAt,});+ if (prepared.replay != null) {+ this.notifyTaskSettled(+ scope,+ prepared.replay.taskId,+ prepared.replay.previousTaskId,+ );+ }if (runtime.signal.aborted) {throw runtime.signal.reason ?? new Error('Subagent task was cancelled.');}@@ -1425,6 +1445,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {}throw new Error(publicFailureDetail(error));} finally {+ if (prepared != null) {+ this.wakeupTaskIds.delete(prepared.replay?.taskId ?? runtime.taskId);+ }if (prepared != null && prepared.replay == null) {this.completeActivity(lease,@@ -2902,6 +2925,10 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {replay: {status: terminal.subagentTask.status as 'completed' | 'error' | 'cancelled',taskId: canonicalTaskId,+ ...(terminal.parentMessageId?.endsWith(':user') === true &&+ terminal.parentMessageId !== `${canonicalTaskId}:user` && {+ previousTaskId: terminal.parentMessageId.slice(0, -':user'.length),+ }),parentRunId: terminal.subagentTask.parentRunId ?? request.parentRunId,content:terminal.text ??@@ -2956,6 +2983,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {status: 'error',content: abandonedMessage,taskId,+ ...(abandoned.messageId.endsWith(':user') && {+ previousTaskId: abandoned.messageId.slice(0, -':user'.length),+ }),parentRunId: request.parentRunId,},};@@ -3192,7 +3222,7 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {) {return;}- await this.onTaskPrepared({+ const admitted = await this.onTaskPrepared({userId: scope.userId,parentConversationId: scope.parentConversationId,parentMessageId: task.parentRunId,@@ -3203,6 +3233,9 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {subagentType: request.subagentType,createdAt: task.createdAt,});+ if (admitted !== false) {+ this.wakeupTaskIds.add(task.taskId);+ }}private async persistCancellation(@@ -3375,6 +3408,26 @@ export class SubagentThreadTaskStore extends InMemorySubagentTaskStore {}logger.error(`[subagentThreads] Failed to refresh ${outcome} child thread`, error);}+ this.notifyTaskSettled(scope, taskId);+ }++ private notifyTaskSettled(+ scope: SubagentThreadScope,+ taskId: string,+ previousTaskId?: string,+ ): void {+ if (!this.wakeupTaskIds.delete(taskId)) {+ return;+ }+ try {+ this.onTaskSettled?.(+ scope.userId,+ scope.parentConversationId,+ previousTaskId == null ? [taskId] : [taskId, previousTaskId],+ );+ } catch (error) {+ logger.warn('[subagentThreads] Settled-task listener failed', error);+ }}private aggregateDetachedUsage(detachedUsage: UsageMetadata[]) {diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.mdindex 875d797218..5fe25e09aa 100644--- a/packages/api/src/agents/triggers/README.md+++ b/packages/api/src/agents/triggers/README.md@@ -103,7 +103,13 @@ experimental clustered startup:`completionWaitMaxIntervalMs` caps how long a background or subagent completion waits betweenreadiness checks while its result or parent turn is not ready. It backs off from 5 seconds by a tenth-of its age, so a long wait costs about one re-check per cap instead of one every five seconds.+of its age; a durable result and a settled generation expedite it, so the cap bounds missed signals.+Parent settlement expedites completion deliveries only in that conversation. A child's durable+terminal result expedites only its task, plus the original task when recovering an abandoned+attempt. Terminal replays signal readiness after registering their delivery, without executing the+child again. A held delivery consumes its wake marker in the same fenced write that releases it+for ordering or defers readiness. These writes use classic operators for DocumentDB compatibility;+older workers can ignore the optional marker and fall back to the configured polling interval.Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-datamigration is needed; optional activity reporting preserves the existing numeric/boolean results.diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.tsindex c5532179a6..697834d4f8 100644--- a/packages/api/src/agents/triggers/engine.spec.ts+++ b/packages/api/src/agents/triggers/engine.spec.ts@@ -123,6 +123,92 @@ describe('createAgentTriggerDeliveryEngine', () => {expect(store.retry).not.toHaveBeenCalled();});+ it('re-dispatches at once a delivery its deferral found expedited while held', async () => {+ const dispatchesWithin = async (deferred: boolean | 'expedited'): Promise<number> => {+ jest.useFakeTimers();+ try {+ /** Models the store: the deferred row is claimable again only when the+ * deferral honored a wake marker and moved it to now. */+ let claimable = true;+ const store = storeWith({+ claimNext: jest.fn(async () => {+ if (!claimable) {+ return null;+ }+ claimable = false;+ return delivery();+ }),+ defer: jest.fn(async () => {+ claimable = deferred === 'expedited';+ return deferred;+ }),+ });+ let dispatches = 0;+ const dispatch = jest.fn(async () => {+ dispatches++;+ if (dispatches > 1) {+ return successResult();+ }+ throw new AgentTriggerExecutionError('The parent generation has not settled yet.', {+ mode: 'continue',+ certainty: 'definite',+ code: 'PARENT_NOT_READY',+ retryable: true,+ deferWithoutAttempt: true,+ retryAfter: '30',+ });+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START, workerId: 'worker-1' },+ { concurrency: 1, tickMs: 60_000, maxIdleTickMs: 60_000 },+ );+ engine.start();+ await jest.advanceTimersByTimeAsync(100);+ await engine.stop();+ return dispatches;+ } finally {+ jest.useRealTimers();+ }+ };++ await expect(dispatchesWithin(true)).resolves.toBe(1);+ await expect(dispatchesWithin('expedited')).resolves.toBe(2);+ });++ it('reclaims an ordering release made due by a held wake signal without waiting for its stale deadline', async () => {+ jest.useFakeTimers();+ try {+ let claimable = true;+ const store = storeWith({+ claimNext: jest.fn(async () => {+ if (!claimable) return null;+ claimable = false;+ return delivery();+ }),+ findEarlierUnsettled: jest+ .fn()+ .mockResolvedValueOnce({ availableAt: new Date(START.getTime() + 60_000) })+ .mockResolvedValue(null),+ release: jest.fn(async () => {+ claimable = true;+ return true;+ }),+ });+ const dispatch = jest.fn(async () => successResult());+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START },+ { concurrency: 1, tickMs: 60_000, maxIdleTickMs: 60_000 },+ );+ engine.start();+ await jest.advanceTimersByTimeAsync(100);+ await engine.stop();+ expect(store.release).toHaveBeenCalledTimes(1);+ expect(dispatch).toHaveBeenCalledTimes(1);+ } finally {+ jest.useRealTimers();+ }+ });+it('runs each claim pass under the root trace context', async () => {const withContext = jest.spyOn(context, 'with');const store = storeWith({ claimNext: jest.fn(async () => null) });diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.tsindex ac5bf350e2..e6d6f9e8fc 100644--- a/packages/api/src/agents/triggers/engine.ts+++ b/packages/api/src/agents/triggers/engine.ts@@ -158,7 +158,7 @@ export interface AgentTriggerDeliveryStore {claimToken: string;attempt: number;availableAt: Date;- }) => Promise<boolean>;+ }) => Promise<boolean | 'expedited'>;complete: (input: {id: string;workerId: string;diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsindex 1130b7f605..0b1278141a 100644--- a/packages/api/src/agents/triggers/service.delivery.spec.ts+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -209,6 +209,143 @@ describe('durable agent trigger service', () => {await service.stop();});+ describe('waiting completion deliveries', () => {+ const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };+ const flush = () => new Promise((resolve) => setImmediate(resolve));+ const result = { status: 'completed' as const, output: 'done', settledAt: START };++ it('expedites and claims the delivery whose background result just became durable', async () => {+ const methods = deliveryMethods({+ persistAgentBackgroundToolResult: jest.fn(async () => true),+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),+ });+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({ address });+ jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear();++ await expect(+ service.persistBackgroundToolResult({+ deliveryKey: 'trigger_background',+ sourceId: 'background-tool-completion',+ result,+ }),+ ).resolves.toBe(true);+ await flush();++ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({+ deliveryKeys: ['trigger_background'],+ sourceIds: ['background-tool-completion', 'subagent-completion'],+ now: expect.any(Date),+ });+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();+ await service.stop();+ });++ it('targets only named subagent completions when a child settles', async () => {+ const methods = deliveryMethods({+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),+ });+ const service = createAgentTriggerService({ methods });+ await service.initialize({ address });+ service.expediteCompletionWakeups({+ user: 'user-1',+ conversationId: 'parent-1',+ taskIds: ['child-1'],+ });+ await flush();+ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({+ user: 'user-1',+ conversationId: 'parent-1',+ taskIds: ['child-1'],+ sourceIds: ['subagent-completion'],+ now: expect.any(Date),+ });+ await service.stop();+ });++ it('does not expedite a result the store refused to persist', async () => {+ const methods = deliveryMethods({+ persistAgentBackgroundToolResult: jest.fn(async () => false),+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),+ });+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({ address });++ await expect(+ service.persistBackgroundToolResult({+ deliveryKey: 'trigger_background',+ sourceId: 'background-tool-completion',+ result,+ }),+ ).resolves.toBe(false);+ await flush();++ expect(methods.expediteAgentTriggerDeliveries).not.toHaveBeenCalled();+ await service.stop();+ });++ it("expedites a principal's completion deliveries when one of its generations settles", async () => {+ const methods = deliveryMethods({+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })),+ });+ let settled: ((event: { userId: string; conversationId: string }) => void) | undefined;+ const unsubscribe = jest.fn();+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ subscribeGenerationSettled: (listener) => {+ settled = listener;+ return unsubscribe;+ },+ });+ await service.initialize({ address });+ jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear();++ settled?.({ userId: '507f1f77bcf86cd799439011', conversationId: 'conversation-1' });+ await flush();++ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledWith({+ user: '507f1f77bcf86cd799439011',+ conversationId: 'conversation-1',+ sourceIds: ['background-tool-completion', 'subagent-completion'],+ now: expect.any(Date),+ });+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();+ await service.stop();+ expect(unsubscribe).toHaveBeenCalledTimes(1);+ });++ it('runs a claim pass even when nothing moved, since a match may already be due', async () => {+ const methods = deliveryMethods({+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 0, held: 0 })),+ });+ let settled: ((event: { userId: string; conversationId: string }) => void) | undefined;+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ subscribeGenerationSettled: (listener) => {+ settled = listener;+ return () => undefined;+ },+ });+ await service.initialize({ address });+ jest.mocked(methods.claimNextAgentTriggerDelivery).mockClear();++ settled?.({ userId: '507f1f77bcf86cd799439011', conversationId: 'conversation-1' });+ await flush();++ expect(methods.expediteAgentTriggerDeliveries).toHaveBeenCalledTimes(1);+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();+ await service.stop();+ });+ });+describe('completion wait configuration', () => {const address = { address: '127.0.0.1', family: 'IPv4' as const, port: 3080 };diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsindex f208336e57..fc2125bef3 100644--- a/packages/api/src/agents/triggers/service.ts+++ b/packages/api/src/agents/triggers/service.ts@@ -26,7 +26,9 @@ import type {import type { AgentTriggerEnqueueOptions, PreparedAgentTriggerDelivery } from './delivery';import type { BoundAddress } from '../../app/origin';import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine';+import { BACKGROUND_TOOL_COMPLETION_SOURCE } from '../backgroundCompletionWakeup';import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown';+import { SUBAGENT_COMPLETION_SOURCE } from '../subagentCompletionWakeup';import { generateAgentTriggerToken } from '../../crypto/jwt';import { prepareAgentTriggerDelivery } from './delivery';import { selfOriginFromAddress } from '../../app/origin';@@ -35,6 +37,9 @@ import { parseAgentTriggerEnvelope } from './envelope';import { createIdleRecoveryLoop } from '../recovery';import { WAITING_RETRY_CAP_MS } from './backoff';+/** Internal sources whose deliveries wait on a result or on their parent generation. */+const COMPLETION_WAKEUP_SOURCES = [BACKGROUND_TOOL_COMPLETION_SOURCE, SUBAGENT_COMPLETION_SOURCE];+export const AGENT_TRIGGER_TOKEN_TTL = '60s';const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;const DEFAULT_USER_DRAIN_POLL_MS = 100;@@ -53,6 +58,11 @@ export interface AgentTriggerServiceOptions {};}+/** A principal's deliveries resuming one conversation, or exact deliveries. */+export type AgentTriggerCompletionExpedite =+ | { user: string; conversationId: string; taskIds?: string[] }+ | { deliveryKeys: string[] };+export interface AgentTriggerServiceDeps {fetch?: AgentTriggerExecutionHostDeps['fetch'];getTimezone?: AgentTriggerExecutionHostDeps['getTimezone'];@@ -69,6 +79,16 @@ export interface AgentTriggerServiceDeps {reclaimCheckpointDeletions?: (limit: number, activity?: { found: boolean }) => Promise<number>;supportsDetachedActionCompletion?: () => boolean;settleSourceBeforeDeadLetter?: AgentTriggerDeliveryEngineDeps['settleSourceBeforeDeadLetter'];+ /** Subscribes to generations reaching a terminal state; returns an unsubscribe. */+ subscribeGenerationSettled?: (+ listener: (event: AgentTriggerGenerationSettledEvent) => void,+ ) => () => void;+}++/** The part of a settled generation that decides which waiting deliveries it may unblock. */+export interface AgentTriggerGenerationSettledEvent {+ userId: string;+ conversationId: string;}export interface AgentTriggerDeliveryReceipt {@@ -124,6 +144,7 @@ export interface AgentTriggerDeliveryPersistence {retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];+ expediteAgentTriggerDeliveries?: AgentTriggerDeliveryMethods['expediteAgentTriggerDeliveries'];getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];@@ -210,6 +231,8 @@ export interface AgentTriggerService {getBackgroundCompletionResultBatchSize: () => number;/** Longest a waiting completion delivery re-checks readiness. */getCompletionWaitMaxIntervalMs: () => number;+ /** Best effort: moves waiting completion deliveries forward after what they wait on changed. */+ expediteCompletionWakeups: (input: AgentTriggerCompletionExpedite) => void;releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];drainUser: (userId: string) => Promise<void>;prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;@@ -539,9 +562,45 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Avoid purgeRecoveryLoop?.start();};+ /** Moves waiting completion deliveries forward when what they wait on has+ * changed, and marks held ones so their next deferral re-checks at once. The+ * claim pass always runs: a matching delivery may already be due here without+ * having moved. Best effort: a missed expedite only means the delivery+ * re-checks at its backoff instead of immediately. */+ const expediteCompletions = (input: AgentTriggerCompletionExpedite): void => {+ const expedite = deps.methods?.expediteAgentTriggerDeliveries;+ if (expedite == null || !deliveryReady || stopping) {+ return;+ }+ void runAsSystem(() =>+ expedite({+ ...('user' in input+ ? {+ user: input.user,+ conversationId: input.conversationId,+ ...(input.taskIds != null && { taskIds: input.taskIds }),+ }+ : { deliveryKeys: input.deliveryKeys }),+ sourceIds:+ 'user' in input && input.taskIds != null+ ? [SUBAGENT_COMPLETION_SOURCE]+ : COMPLETION_WAKEUP_SOURCES,+ now: new Date(),+ }),+ )+ .then(() => deliveryEngine?.wake())+ .catch((error) =>+ logger.warn('[agent-triggers] failed to expedite waiting completion deliveries:', error),+ );+ };++ let unsubscribeGenerationSettled: (() => void) | undefined;+const stop = async (): Promise<void> => {stopping = true;deliveryReady = false;+ unsubscribeGenerationSettled?.();+ unsubscribeGenerationSettled = undefined;await purgeRecoveryLoop?.stop();await initializePromise?.catch(() => undefined);await deliveryEngine?.stop();@@ -611,6 +670,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A);deliveryReady = true;deliveryEngine.start();+ /** Deliveries waiting on a parent resume that parent's conversation, so a+ * settled generation wakes only those, not every waiting task of the user. */+ unsubscribeGenerationSettled ??= deps.subscribeGenerationSettled?.(+ ({ userId, conversationId }) => expediteCompletions({ user: userId, conversationId }),+ );startPurgeRecovery();logger.info('[agent-triggers] durable delivery engine started');}).finally(() => {@@ -727,7 +791,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): ApersistBackgroundToolResult: (input) =>runAsSystem(async () => {const persist = requireMethods().persistAgentBackgroundToolResult;- return persist == null ? false : persist(input);+ const persisted = persist == null ? false : await persist(input);+ if (persisted) {+ expediteCompletions({ deliveryKeys: [input.deliveryKey] });+ }+ return persisted;}),getBackgroundToolResultClaim: (input) =>runAsSystem(async () => {@@ -736,6 +804,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}),getBackgroundCompletionResultBatchSize: () => backgroundCompletionResultBatchSize,getCompletionWaitMaxIntervalMs: () => completionWaitMaxIntervalMs,+ expediteCompletionWakeups: (input) => expediteCompletions(input),releaseBackgroundToolResultClaims: (input) =>runAsSystem(async () => {const release = requireMethods().releaseAgentBackgroundToolResultClaims;diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 991ee1b882..a92318e29f 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -568,10 +568,22 @@ export interface CreateGenerationJobOptions {* receiving this claim, then pass the same object to {@link finishTerminalJob}* from a `finally` block.*/+/** A generation reached a terminal state and released its runtime. */+export interface GenerationSettledEvent {+ streamId: string;+ conversationId: string;+ userId: string;+ status: TerminalJobClaim['status'];+}++export type GenerationSettledListener = (event: GenerationSettledEvent) => void;+export interface TerminalJobClaim {readonly streamId: string;readonly createdAt: number;readonly conversationId?: string;+ /** The generation's owner, so settlement can be announced to that principal's waiters. */+ readonly userId?: string;readonly status: 'complete' | 'error' | 'aborted';readonly error?: string;/** The winner must durably publish either its normal FINAL or a@@ -786,6 +798,7 @@ class GenerationJobManagerClass {/** Makes terminal cleanup idempotent while keeping claims opaque to callers. */private terminalFinishPromises = new WeakMap<TerminalJobClaim, Promise<void>>();+ private generationSettledListeners = new Set<GenerationSettledListener>();/** Exact local runtime observed when a claim won; never clean a later runtime. */private terminalClaimRuntimes = new WeakMap<TerminalJobClaim, RuntimeJobState | null>();@@ -1078,6 +1091,7 @@ class GenerationJobManagerClass {createdAt,...(job?.createdAt === createdAt &&job.conversationId != null && { conversationId: job.conversationId }),+ ...(job?.createdAt === createdAt && job.userId != null && { userId: job.userId }),status: 'error' as const,error,drainedSteers: Object.freeze([...drainedSteers]),@@ -2256,7 +2270,7 @@ class GenerationJobManagerClass {streamId: string,job: Pick<SerializableJobData,- 'createdAt' | 'conversationId' | 'providerExecutionId' | 'agentEventDeliveryKey'+ 'createdAt' | 'conversationId' | 'providerExecutionId' | 'agentEventDeliveryKey' | 'userId'>,message: string,): Promise<boolean> {@@ -2288,6 +2302,13 @@ class GenerationJobManagerClass {expectCreatedAt: job.createdAt,})) {+ /** A direct terminal transition builds no claim, so it announces itself. */+ this.notifyGenerationSettled({+ streamId,+ conversationId: job.conversationId,+ userId: job.userId,+ status: 'error',+ });return true;}} catch (error) {@@ -2298,6 +2319,15 @@ class GenerationJobManagerClass {try {const current = await this.jobStore.getJob(streamId);+ if (current?.createdAt === job.createdAt && current.status === 'error') {+ this.notifyGenerationSettled({+ streamId,+ conversationId: current.conversationId,+ userId: current.userId,+ status: 'error',+ });+ return true;+ }if (current == null ||current.createdAt !== job.createdAt ||@@ -3901,6 +3931,7 @@ class GenerationJobManagerClass {...(jobData.conversationId != null && {conversationId: jobData.conversationId,}),+ ...(jobData.userId != null && { userId: jobData.userId }),status,...(terminalError != null && { error: terminalError }),...(options.persistencePending === true && {@@ -4071,6 +4102,40 @@ class GenerationJobManagerClass {* claim is idempotent, and every local mutation is pinned to the runtime* object and generation epoch captured when the CAS won.*/+ /**+ * Calls `listener` after each generation owned by this process reaches a+ * terminal state and its runtime is released — by completion, error, or+ * abort. Listeners run synchronously and must not throw; failures are logged+ * and never affect terminal cleanup. Returns an unsubscribe function.+ */+ onGenerationSettled(listener: GenerationSettledListener): () => void {+ this.generationSettledListeners.add(listener);+ return () => {+ this.generationSettledListeners.delete(listener);+ };+ }++ private notifyGenerationSettled(+ target: Pick<TerminalJobClaim, 'streamId' | 'conversationId' | 'userId' | 'status'>,+ ): void {+ if (target.userId == null || this.generationSettledListeners.size === 0) {+ return;+ }+ const event: GenerationSettledEvent = {+ streamId: target.streamId,+ conversationId: target.conversationId ?? target.streamId,+ userId: target.userId,+ status: target.status,+ };+ for (const listener of this.generationSettledListeners) {+ try {+ listener(event);+ } catch (listenerError) {+ logger.error('[GenerationJobManager] Generation settled listener failed', listenerError);+ }+ }+ }+finishTerminalJob(claim: TerminalJobClaim): Promise<void> {const inFlight = this.terminalFinishPromises.get(claim);if (inFlight) {@@ -4260,6 +4325,7 @@ class GenerationJobManagerClass {metricStatus = 'error';}recordGenerationJob(this.storeLabel, metricStatus);+ this.notifyGenerationSettled(claim);}if (cleanupError != null) {@@ -4616,6 +4682,7 @@ class GenerationJobManagerClass {...(jobData.conversationId != null && {conversationId: jobData.conversationId,}),+ ...(jobData.userId != null && { userId: jobData.userId }),status: 'aborted',persistencePending: true,drainedSteers: Object.freeze([...drainedSteers]),@@ -8865,6 +8932,17 @@ class GenerationJobManagerClass {await this.runApprovalExpiredHandler(streamId, expiredJob);await this.notifyApprovalExpiredRuntime(streamId, expiredJob.createdAt, observedRuntime);+ /** Expiry is a direct `requires_action -> aborted` transition that never builds a+ * terminal claim, so it announces settlement itself. */+ this.notifyGenerationSettled({+ streamId,+ conversationId: expiredJob.conversationId,+ userId: expiredJob.userId,+ status: 'aborted',+ });+ /** Terminal now; releasing ownership keeps the sweep's relay branch from+ * announcing this generation a second time. */+ this.releaseJobOwnership(streamId, expiredJob.createdAt);return true;}@@ -9055,7 +9133,17 @@ class GenerationJobManagerClass {await this.runApprovalExpiredHandler(streamId, job);}await this.notifyApprovalExpiredRuntime(streamId, job.createdAt, runtime);- changed = this.releaseJobOwnership(streamId, job.createdAt) || changed;+ const released = this.releaseJobOwnership(streamId, job.createdAt);+ if (released) {+ /** The store won the expiry CAS, so no local claim announced it. */+ this.notifyGenerationSettled({+ streamId,+ conversationId: job.conversationId,+ userId: job.userId,+ status: 'aborted',+ });+ }+ changed = released || changed;continue;}if (diff --git a/packages/api/src/stream/__tests__/generationSettled.spec.ts b/packages/api/src/stream/__tests__/generationSettled.spec.tsnew file mode 100644index 0000000000..473b548dce--- /dev/null+++ b/packages/api/src/stream/__tests__/generationSettled.spec.ts@@ -0,0 +1,145 @@+/**+ * Waiting completion deliveries are expedited when a generation settles, so the+ * settled notification must fire exactly once for every terminal path and name+ * the generation's owner — and a failing listener must never disturb cleanup.+ */+import type { GenerationSettledEvent } from '../GenerationJobManager';+import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';++/** Suppress winston Console transport output (survives jest.resetModules) */+jest.spyOn(console, 'log').mockImplementation();++async function configureManager() {+ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { InMemoryJobStore } = await import('../implementations/InMemoryJobStore');+ const { InMemoryEventTransport } = await import('../implementations/InMemoryEventTransport');++ GenerationJobManager.configure({+ jobStore: new InMemoryJobStore(),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ GenerationJobManager.initialize();+ return GenerationJobManager;+}++describe('generation settled notifications', () => {+ beforeEach(() => {+ jest.resetModules();+ });++ it('announces a completed generation once, with its owner', async () => {+ const manager = await configureManager();+ const events: GenerationSettledEvent[] = [];+ manager.onGenerationSettled((event) => events.push(event));+ const job = await manager.createJob('settled-complete', 'user-1', 'conversation-1');++ await manager.completeJob('settled-complete', undefined, job.createdAt);+ await manager.completeJob('settled-complete', undefined, job.createdAt);++ expect(events).toEqual([+ {+ streamId: 'settled-complete',+ conversationId: 'conversation-1',+ userId: 'user-1',+ status: 'complete',+ },+ ]);+ await manager.destroy();+ });++ it('announces a failed generation', async () => {+ const manager = await configureManager();+ const events: GenerationSettledEvent[] = [];+ manager.onGenerationSettled((event) => events.push(event));+ const job = await manager.createJob('settled-error', 'user-2');++ await manager.completeJob('settled-error', 'provider failed', job.createdAt);++ expect(events).toEqual([+ expect.objectContaining({ streamId: 'settled-error', userId: 'user-2', status: 'error' }),+ ]);+ await manager.destroy();+ });++ it('announces an aborted generation', async () => {+ const manager = await configureManager();+ const events: GenerationSettledEvent[] = [];+ manager.onGenerationSettled((event) => events.push(event));+ await manager.createJob('settled-abort', 'user-3');++ const result = await manager.abortJob('settled-abort');++ expect(result.success).toBe(true);+ expect(events).toEqual([+ expect.objectContaining({ streamId: 'settled-abort', userId: 'user-3', status: 'aborted' }),+ ]);+ await manager.destroy();+ });++ it('announces a generation whose approval expired', async () => {+ const manager = await configureManager();+ const events: GenerationSettledEvent[] = [];+ manager.onGenerationSettled((event) => events.push(event));+ await manager.createJob('settled-approval', 'user-6', 'conversation-6');+ const action = buildPendingAction(+ buildToolApprovalPayload([+ { name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_expire' },+ ]),+ {+ streamId: 'settled-approval',+ conversationId: 'conversation-6',+ runId: 'run-1',+ responseMessageId: 'msg-1',+ },+ );+ await manager.approvals.pause('settled-approval', action);+ expect(events).toEqual([]);++ await expect(manager.expireApproval('settled-approval')).resolves.toBe(true);+ await expect(manager.expireApproval('settled-approval')).resolves.toBe(false);+ /** The next expiry sweep relays the terminal job without announcing it again. */+ await (manager as unknown as { cleanup(): Promise<void> }).cleanup();++ expect(events).toEqual([+ {+ streamId: 'settled-approval',+ conversationId: 'conversation-6',+ userId: 'user-6',+ status: 'aborted',+ },+ ]);+ await manager.destroy();+ });++ it('keeps cleanup and later listeners running when one listener throws', async () => {+ const manager = await configureManager();+ const events: GenerationSettledEvent[] = [];+ manager.onGenerationSettled(() => {+ throw new Error('listener failed');+ });+ manager.onGenerationSettled((event) => events.push(event));+ const job = await manager.createJob('settled-throwing-listener', 'user-4');++ await expect(+ manager.completeJob('settled-throwing-listener', undefined, job.createdAt),+ ).resolves.toBe(true);++ expect(events).toHaveLength(1);+ await manager.destroy();+ });++ it('stops notifying after unsubscribe', async () => {+ const manager = await configureManager();+ const listener = jest.fn();+ const unsubscribe = manager.onGenerationSettled(listener);+ unsubscribe();+ const job = await manager.createJob('settled-unsubscribed', 'user-5');++ await manager.completeJob('settled-unsubscribed', undefined, job.createdAt);++ expect(listener).not.toHaveBeenCalled();+ await manager.destroy();+ });+});diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.tsindex 3b5d964d9d..1c8578d86d 100644--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts@@ -1070,48 +1070,86 @@ describe('GenerationJobManager start-generation claim', () => {expect(Object.getOwnPropertyDescriptor(durable!, 'replacedJobs')).toBeUndefined();});- it('terminalizes the exact committed epoch when legacy verification fails during recovery', async () => {- const streamId = 'stream-lost-create-legacy-failure';- const clientRequestId = 'req-lost-create-legacy-failure';- const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);- const actualCreate = store.createJob.bind(store);- jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {- await actualCreate(...args);- throw new Error('simulated lost atomic create reply');- });- const actualClaim = store.claimIdempotencyKey.bind(store);- let failLegacyProbe = true;- jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => {- if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) {- failLegacyProbe = false;- return Promise.reject(new Error('simulated legacy probe outage'));- }- return actualClaim(key, value, ttlSeconds);- });+ it.each([+ { loseTerminalReply: false, replaced: false },+ { loseTerminalReply: true, replaced: false },+ { loseTerminalReply: true, replaced: true },+ ])(+ 'announces only the exact recovered epoch (lost reply: $loseTerminalReply, replaced: $replaced)',+ async ({ loseTerminalReply, replaced }) => {+ const settled = jest.fn();+ manager.onGenerationSettled(settled);+ const transition = store.transitionStatus.bind(store);+ jest.spyOn(store, 'transitionStatus').mockImplementation(async (id, input) => {+ const committed = await transition(id, input);+ if (loseTerminalReply && input.to === 'error' && committed) {+ if (replaced) {+ const previous = await store.getJob(id);+ const replacement = await actualCreate(id, 'user-1', id);+ await transition(id, {+ from: 'running',+ to: 'error',+ expectCreatedAt: replacement.createdAt,+ patch: { error: previous?.error, finalEvent: previous?.finalEvent },+ });+ }+ throw new Error('lost terminal CAS reply');+ }+ return committed;+ });+ const streamId = 'stream-lost-create-legacy-failure';+ const clientRequestId = 'req-lost-create-legacy-failure';+ const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);+ const actualCreate = store.createJob.bind(store);+ jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {+ await actualCreate(...args);+ throw new Error('simulated lost atomic create reply');+ });+ const actualClaim = store.claimIdempotencyKey.bind(store);+ let failLegacyProbe = true;+ jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => {+ if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) {+ failLegacyProbe = false;+ return Promise.reject(new Error('simulated legacy probe outage'));+ }+ return actualClaim(key, value, ttlSeconds);+ });- await expect(- manager.createJob(streamId, 'user-1', streamId, {- idempotencyClientRequestId: clientRequestId,- idempotencyClaimToken: claim.existing!.claimToken,- initialMetadata: { generationProtocolVersion: 2 },- }),- ).rejects.toThrow('simulated lost atomic create reply');+ await expect(+ manager.createJob(streamId, 'user-1', streamId, {+ idempotencyClientRequestId: clientRequestId,+ idempotencyClaimToken: claim.existing!.claimToken,+ initialMetadata: { generationProtocolVersion: 2 },+ }),+ ).rejects.toThrow('simulated lost atomic create reply');- const durable = await store.getJob(streamId);- expect(durable).toMatchObject({- status: 'error',- error: 'Generation idempotency rollout fence could not be recovered',- finalEvent: expect.stringContaining('terminal_payload_missing'),- });- const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);- expect(retry).toMatchObject({- claimed: false,- existing: { startedAt: durable!.createdAt },- });- await expect(- manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),- ).resolves.toMatchObject({ claimed: false });- });+ const durable = await store.getJob(streamId);+ expect(durable).toMatchObject({+ status: 'error',+ error: 'Generation idempotency rollout fence could not be recovered',+ finalEvent: expect.stringContaining('terminal_payload_missing'),+ });+ if (replaced) {+ expect(settled).not.toHaveBeenCalled();+ return;+ }+ expect(settled).toHaveBeenCalledTimes(1);+ expect(settled).toHaveBeenCalledWith({+ streamId,+ conversationId: streamId,+ userId: 'user-1',+ status: 'error',+ });+ const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);+ expect(retry).toMatchObject({+ claimed: false,+ existing: { startedAt: durable!.createdAt },+ });+ await expect(+ manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),+ ).resolves.toMatchObject({ claimed: false });+ },+ );it('terminalizes and preserves the primary fence when the legacy started mark fails', async () => {const claim = await manager.claimGeneration(diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.tsindex b6c97e6098..ef72b19ee5 100644--- a/packages/api/src/stream/index.ts+++ b/packages/api/src/stream/index.ts@@ -4,6 +4,8 @@ export {type CreateGenerationJobOptions,type GenerationJobManagerOptions,type TerminalJobClaim,+ type GenerationSettledEvent,+ type GenerationSettledListener,GENERATION_RECOVERY_FAILED_ERROR,TERMINAL_PUBLICATION_RECONNECT_ERROR,} from './GenerationJobManager';diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.tsindex 3c6850d7b3..7498b932c4 100644--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts@@ -453,6 +453,348 @@ describe('agent trigger delivery methods', () => {).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil });});+ describe('expediteAgentTriggerDeliveries', () => {+ const background = { id: 'background-tool-completion', type: 'internal' };+ const later = new Date(START.getTime() + 30_000);+ const capable = {+ workerId: 'background-capable-worker',+ claimToken: 'background-capable-claim',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],+ };+ const waiting = (overrides: Partial<Parameters<typeof enqueueInput>[0]> = {}) =>+ methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ orderingKey: `background-lane-${counter + 1}`,+ envelope: { event: { source: background } },+ requiredWorkerCapability:+ AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,+ availableAt: later,+ ...overrides,+ }),+ );++ it('makes a deferred delivery claimable now once its result is durable', async () => {+ const target = await waiting();+ const sibling = await waiting();+ await expect(methods.claimNextAgentTriggerDelivery(capable)).resolves.toBeNull();++ await expect(+ methods.expediteAgentTriggerDeliveries({+ deliveryKeys: [target.delivery.deliveryKey],+ sourceIds: [background.id],+ now: START,+ }),+ ).resolves.toEqual({ expedited: 1, held: 0 });++ await expect(methods.claimNextAgentTriggerDelivery(capable)).resolves.toMatchObject({+ id: target.delivery.id,+ });+ await expect(+ methods.claimNextAgentTriggerDelivery({ ...capable, claimToken: 'second-claim' }),+ ).resolves.toBeNull();+ const untouched = await Delivery.findById(sibling.delivery.id).lean();+ expect(untouched?.availableAt).toEqual(later);+ });++ it("moves only the principal's waiting rows from the named sources", async () => {+ const user = new mongoose.Types.ObjectId();+ const mine = await waiting({ user });+ const otherUser = await waiting();+ const otherSource = await waiting({+ user,+ envelope: { event: { source: { id: 'agent-queued-turn', type: 'internal' } } },+ });+ const external = await waiting({+ user,+ envelope: { event: { source: { id: background.id, type: 'webhook' } } },+ });+ const due = await waiting({ user, availableAt: START });++ await expect(+ methods.expediteAgentTriggerDeliveries({ user, sourceIds: [background.id], now: START }),+ ).resolves.toEqual({ expedited: 1, held: 0 });++ const rows = await Delivery.find({+ _id: {+ $in: [mine, otherUser, otherSource, external, due].map((row) => row.delivery.id),+ },+ }).lean();+ const availableAt = new Map(rows.map((row) => [String(row._id), row.availableAt]));+ expect(availableAt.get(mine.delivery.id)).toEqual(START);+ expect(availableAt.get(otherUser.delivery.id)).toEqual(later);+ expect(availableAt.get(otherSource.delivery.id)).toEqual(later);+ expect(availableAt.get(external.delivery.id)).toEqual(later);+ expect(availableAt.get(due.delivery.id)).toEqual(START);+ });++ it('marks a delivery a worker currently holds instead of moving it', async () => {+ const user = new mongoose.Types.ObjectId();+ const held = await waiting({ user, availableAt: START });+ const claim = await methods.claimNextAgentTriggerDelivery(capable);+ expect(claim).toMatchObject({ id: held.delivery.id });+ const before = await Delivery.findById(held.delivery.id).lean();++ await expect(+ methods.expediteAgentTriggerDeliveries({+ user,+ sourceIds: [background.id],+ now: new Date(START.getTime() - 60_000),+ }),+ ).resolves.toEqual({ expedited: 0, held: 1 });++ const after = await Delivery.findById(held.delivery.id).lean();+ expect(after?.status).toBe(before?.status);+ expect(after?.availableAt).toEqual(before?.availableAt);+ expect(after?.wakeRequestedAt).toEqual(new Date(START.getTime() - 60_000));+ });++ it('re-checks at once when readiness changed while the delivery was held', async () => {+ const user = new mongoose.Types.ObjectId();+ const held = await waiting({ user, availableAt: START });+ const claim = await methods.claimNextAgentTriggerDelivery(capable);+ expect(claim).toMatchObject({ id: held.delivery.id });+ const fence = {+ id: held.delivery.id,+ workerId: capable.workerId,+ claimToken: capable.claimToken,+ };+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...fence, now: START });+ await expect(+ methods.expediteAgentTriggerDeliveries({ user, sourceIds: [background.id], now: START }),+ ).resolves.toEqual({ expedited: 0, held: 1 });++ const beforeDefer = Date.now();+ await expect(+ methods.deferAgentTriggerDeliveryAttempt({+ ...fence,+ attempt: attempt!,+ availableAt: later,+ }),+ ).resolves.toBe('expedited');++ const deferred = await Delivery.findById(held.delivery.id).lean();+ expect(deferred?.wakeRequestedAt).toBeUndefined();+ expect(deferred?.availableAt.getTime()).toBeGreaterThanOrEqual(beforeDefer);+ expect(deferred?.availableAt).not.toEqual(later);++ const reclaimed = await methods.claimNextAgentTriggerDelivery({+ ...capable,+ claimToken: 'second-claim',+ now: new Date(),+ leaseUntil: new Date(Date.now() + 60_000),+ });+ expect(reclaimed).toMatchObject({ id: held.delivery.id });+ const secondFence = { ...fence, claimToken: 'second-claim' };+ const secondAttempt = await methods.beginAgentTriggerDeliveryAttempt({+ ...secondFence,+ now: new Date(),+ });+ await expect(+ methods.deferAgentTriggerDeliveryAttempt({+ ...secondFence,+ attempt: secondAttempt!,+ availableAt: later,+ }),+ ).resolves.toBe(true);+ expect((await Delivery.findById(held.delivery.id).lean())?.availableAt).toEqual(later);+ });++ it.each(['ordinary', 'legacy', 'shielded'] as const)(+ 'honors a held wake marker on ordering release for %s leases',+ async (profile) => {+ const row = await waiting({+ availableAt: START,+ ...(profile === 'ordinary' && { requiredWorkerCapability: undefined }),+ });+ if (profile === 'legacy') {+ await Delivery.updateOne(+ { _id: row.delivery.id },+ {+ $set: { status: 'capability_pending', availableAt: START },+ $unset: { capabilityStatus: 1, leaseUntil: 1 },+ },+ );+ }+ const claimed = await methods.claimNextAgentTriggerDelivery(capable);+ expect(claimed?.id).toBe(row.delivery.id);+ const fence = {+ id: row.delivery.id,+ workerId: capable.workerId,+ claimToken: capable.claimToken,+ };+ await methods.expediteAgentTriggerDeliveries({+ deliveryKeys: [row.delivery.deliveryKey],+ sourceIds: [background.id],+ now: START,+ });+ await expect(+ methods.releaseAgentTriggerDelivery({+ ...fence,+ claimToken: 'stale',+ availableAt: later,+ }),+ ).resolves.toBe(false);+ expect((await Delivery.findById(row.delivery.id).lean())?.wakeRequestedAt).toEqual(START);+ await expect(+ methods.releaseAgentTriggerDelivery({ ...fence, availableAt: later }),+ ).resolves.toBe(true);+ const released = await Delivery.findById(row.delivery.id).lean();+ expect(released?.wakeRequestedAt).toBeUndefined();+ expect(released?.availableAt.getTime()).toBeLessThanOrEqual(Date.now());+ expect(released?.claimAvailableAt).toEqual(released?.availableAt);+ expect(released?.attempts).toBe(0);+ const reclaimed = await methods.claimNextAgentTriggerDelivery({+ ...capable,+ now: new Date(),+ claimToken: 'next',+ });+ expect(reclaimed?.id).toBe(row.delivery.id);+ await methods.releaseAgentTriggerDelivery({+ ...fence,+ claimToken: 'next',+ availableAt: later,+ });+ expect((await Delivery.findById(row.delivery.id).lean())?.availableAt).toEqual(later);+ },+ );++ it.each(['release', 'defer'] as const)(+ 'does not lose an expedite between the unmarked and marked %s writes',+ async (operation) => {+ const row = await waiting({ availableAt: START });+ await methods.claimNextAgentTriggerDelivery(capable);+ const fence = {+ id: row.delivery.id,+ workerId: capable.workerId,+ claimToken: capable.claimToken,+ };+ const attempt =+ operation === 'defer'+ ? await methods.beginAgentTriggerDeliveryAttempt({ ...fence, now: START })+ : undefined;+ const updateOne = Delivery.collection.updateOne.bind(Delivery.collection);+ let injected = false;+ const spy = jest+ .spyOn(Delivery.collection, 'updateOne')+ .mockImplementation(async (filter, update, options) => {+ if (+ !injected &&+ filter.capabilityClaimToken === capable.claimToken &&+ (filter.wakeRequestedAt as { $exists?: boolean } | undefined)?.$exists === false+ ) {+ injected = true;+ await methods.expediteAgentTriggerDeliveries({+ deliveryKeys: [row.delivery.deliveryKey],+ sourceIds: [background.id],+ now: START,+ });+ }+ return updateOne(filter, update, options);+ });+ try {+ const input = { ...fence, availableAt: later };+ if (operation === 'defer') {+ await expect(+ methods.deferAgentTriggerDeliveryAttempt({ ...input, attempt: attempt! }),+ ).resolves.toBe('expedited');+ } else {+ await expect(methods.releaseAgentTriggerDelivery(input)).resolves.toBe(true);+ }+ } finally {+ spy.mockRestore();+ }+ expect(injected).toBe(true);+ const released = await Delivery.findById(row.delivery.id).lean();+ expect(released?.wakeRequestedAt).toBeUndefined();+ expect(released?.availableAt.getTime()).toBeLessThanOrEqual(Date.now());+ expect(released?.attempts).toBe(0);+ },+ );++ it('leaves unfinished siblings backed off when only named child tasks settle', async () => {+ const user = new mongoose.Types.ObjectId();+ const source = { id: 'subagent-completion', type: 'internal' };+ const task = (taskId: string) =>+ waiting({+ user,+ envelope: {+ event: { source, payload: { taskId } },+ target: { conversationId: 'parent' },+ },+ });+ const original = await task('original');+ const recovered = await task('recovered');+ const sibling = await task('sibling');+ await expect(+ methods.expediteAgentTriggerDeliveries({+ user,+ conversationId: 'parent',+ taskIds: ['original', 'recovered'],+ sourceIds: [source.id],+ now: START,+ }),+ ).resolves.toEqual({ expedited: 2, held: 0 });+ expect((await Delivery.findById(original.delivery.id).lean())?.availableAt).toEqual(START);+ expect((await Delivery.findById(recovered.delivery.id).lean())?.availableAt).toEqual(START);+ expect((await Delivery.findById(sibling.delivery.id).lean())?.availableAt).toEqual(later);+ await expect(+ methods.expediteAgentTriggerDeliveries({+ user,+ conversationId: 'parent',+ taskIds: [],+ sourceIds: [source.id],+ now: START,+ }),+ ).rejects.toThrow(TypeError);+ });++ it("narrows a principal's selection to the conversation being resumed", async () => {+ const user = new mongoose.Types.ObjectId();+ const target = await waiting({+ user,+ envelope: { event: { source: background }, target: { conversationId: 'settled-convo' } },+ });+ const other = await waiting({+ user,+ envelope: { event: { source: background }, target: { conversationId: 'other-convo' } },+ });++ await expect(+ methods.expediteAgentTriggerDeliveries({+ user,+ conversationId: 'settled-convo',+ sourceIds: [background.id],+ now: START,+ }),+ ).resolves.toEqual({ expedited: 1, held: 0 });++ expect((await Delivery.findById(target.delivery.id).lean())?.availableAt).toEqual(START);+ expect((await Delivery.findById(other.delivery.id).lean())?.availableAt).toEqual(later);+ });++ it('refuses an unbounded or malformed selection', async () => {+ await expect(+ methods.expediteAgentTriggerDeliveries({ sourceIds: [background.id], now: START }),+ ).rejects.toThrow(TypeError);+ await expect(+ methods.expediteAgentTriggerDeliveries({+ user: new mongoose.Types.ObjectId(),+ sourceIds: [],+ now: START,+ }),+ ).rejects.toThrow(TypeError);+ await expect(+ methods.expediteAgentTriggerDeliveries({+ deliveryKeys: ['trigger_x'],+ sourceIds: [background.id],+ now: new Date(Number.NaN),+ }),+ ).rejects.toThrow(TypeError);+ });+ });+it('persists one private background result receipt independently of message rows', async () => {const source = { id: 'background-tool-completion', type: 'internal' };const queued = await methods.enqueueAgentTriggerDelivery(@@ -2468,7 +2810,7 @@ describe('agent trigger delivery methods', () => {);});- it("indexes a user's waiting deliveries for user-scoped readiness reads", async () => {+ it("indexes a user's waiting deliveries for settle-time expediting", async () => {const deliveryIndexes = await Delivery.collection.indexes();const userIndex = deliveryIndexes.find((index) =>diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.tsindex e3fe4c2d0f..1e2b6ead6f 100644--- a/packages/data-schemas/src/methods/triggerDelivery.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.ts@@ -246,6 +246,27 @@ export interface AgentEventActorReceiptStorageMetrics {deadDeliveries: number;}+/** Selects deferred internal deliveries whose readiness condition just changed. */+export interface ExpediteAgentTriggerDeliveriesInput {+ sourceIds: readonly string[];+ /** Exact deliveries, e.g. the one whose result just became durable. */+ deliveryKeys?: readonly string[];+ /** Every matching delivery of one principal, e.g. after one of its generations settled. */+ user?: string | Types.ObjectId;+ /** Narrows a principal's selection to deliveries that resume this conversation. */+ conversationId?: string;+ /** Exact tasks within a principal's conversation, including a repaired attempt's predecessor. */+ taskIds?: readonly string[];+ now: Date;+}++export interface ExpediteAgentTriggerDeliveriesResult {+ /** Deferred, unheld deliveries moved to `now`. */+ expedited: number;+ /** Deliveries a worker held, marked so their next deferral re-checks at once. */+ held: number;+}+export interface AgentTriggerDeliveryMethods {ensureAgentTriggerDeliveryIndexes: () => Promise<void>;enqueueAgentTriggerDelivery: (@@ -270,9 +291,10 @@ export interface AgentTriggerDeliveryMethods {beginAgentTriggerDeliveryAttempt: (input: AgentTriggerDeliveryFence & { now: Date },) => Promise<number | null>;+ /** `expedited` when readiness changed while the delivery was held: it is due now instead. */deferAgentTriggerDeliveryAttempt: (input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },- ) => Promise<boolean>;+ ) => Promise<boolean | 'expedited'>;completeAgentTriggerDelivery: (input: AgentTriggerDeliveryFence & {attempt: number;@@ -307,6 +329,9 @@ export interface AgentTriggerDeliveryMethods {sourceId: string;now: Date;}) => Promise<AgentTriggerProducerLeaseStatus>;+ expediteAgentTriggerDeliveries: (+ input: ExpediteAgentTriggerDeliveriesInput,+ ) => Promise<ExpediteAgentTriggerDeliveriesResult>;persistAgentBackgroundToolResult: (input: PersistAgentBackgroundToolResultInput,) => Promise<boolean>;@@ -1846,57 +1871,61 @@ export function createAgentTriggerDeliveryMethods($or: [ordinaryFence(input), legacyCapabilityFence(input), shieldCapabilityFence(input)],});+ /** Readiness signals and the lease release must meet in one fenced write. Try+ * the unmarked state first, then the marked state: a concurrent expedite can+ * only add a marker while this lease is held. After release, expedite's second+ * update sees an unheld row. No pipeline updates or unfenced follow-up needed. */+ async function releaseWaitingDelivery(+ input: AgentTriggerDeliveryFence & { availableAt: Date; attempt?: number },+ ): Promise<boolean | 'expedited'> {+ const attemptFence = input.attempt == null ? {} : { attempts: input.attempt };+ const attemptChange = input.attempt == null ? {} : { $inc: { attempts: -1 } };+ const profiles = [+ { filter: shieldCapabilityFence(input), status: 'leased', capabilityStatus: 'pending' },+ { filter: legacyCapabilityFence(input), status: 'capability_pending' },+ { filter: ordinaryFence(input), status: 'pending' },+ ] as const;+ for (const marked of [false, true]) {+ for (const profile of profiles) {+ const availableAt = marked ? new Date() : input.availableAt;+ const result = await Delivery().updateOne(+ {+ _id: input.id,+ ...profile.filter,+ ...attemptFence,+ wakeRequestedAt: { $exists: marked },+ },+ {+ ...attemptChange,+ $set: {+ status: profile.status,+ ...('capabilityStatus' in profile && { capabilityStatus: profile.capabilityStatus }),+ availableAt,+ claimAvailableAt: availableAt,+ },+ $unset: {+ leaseBy: 1,+ leaseUntil: 1,+ claimToken: 1,+ capabilityLeaseBy: 1,+ capabilityLeaseUntil: 1,+ capabilityClaimToken: 1,+ wakeRequestedAt: 1,+ },+ },+ );+ if (result.modifiedCount === 1) {+ return marked ? 'expedited' : true;+ }+ }+ }+ return false;+ }+async function releaseAgentTriggerDelivery(input: AgentTriggerDeliveryFence & { availableAt: Date },): Promise<boolean> {- const shieldResult = await Delivery().updateOne(- { _id: input.id, ...shieldCapabilityFence(input) },- {- $set: {- status: 'leased',- availableAt: input.availableAt,- capabilityStatus: 'pending',- claimAvailableAt: input.availableAt,- },- $unset: {- leaseBy: 1,- leaseUntil: 1,- claimToken: 1,- capabilityLeaseBy: 1,- capabilityLeaseUntil: 1,- capabilityClaimToken: 1,- },- },- );- if (shieldResult.modifiedCount === 1) {- return true;- }- const capabilityResult = await Delivery().updateOne(- { _id: input.id, ...legacyCapabilityFence(input) },- {- $set: {- status: 'capability_pending',- availableAt: input.availableAt,- claimAvailableAt: input.availableAt,- },- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },- },- );- if (capabilityResult.modifiedCount === 1) {- return true;- }- const result = await Delivery().updateOne(- { _id: input.id, ...ordinaryFence(input) },- {- $set: {- status: 'pending',- availableAt: input.availableAt,- claimAvailableAt: input.availableAt,- },- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },- },- );- return result.modifiedCount === 1;+ return (await releaseWaitingDelivery(input)) !== false;}async function beginAgentTriggerDeliveryAttempt(@@ -1926,64 +1955,11 @@ export function createAgentTriggerDeliveryMethods(/** Releases a pre-dispatch deferral and restores the attempt consumed by beginAttempt. */async function deferAgentTriggerDeliveryAttempt(input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },- ): Promise<boolean> {+ ): Promise<boolean | 'expedited'> {if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) {throw new TypeError('attempt must be a positive integer');}- const update = {- $inc: { attempts: -1 },- $set: {- availableAt: input.availableAt,- claimAvailableAt: input.availableAt,- },- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },- };- const shieldResult = await Delivery().updateOne(- {- _id: input.id,- ...shieldCapabilityFence(input),- attempts: input.attempt,- },- {- $inc: update.$inc,- $set: {- status: 'leased',- availableAt: input.availableAt,- capabilityStatus: 'pending',- claimAvailableAt: input.availableAt,- },- $unset: {- leaseBy: 1,- leaseUntil: 1,- claimToken: 1,- capabilityLeaseBy: 1,- capabilityLeaseUntil: 1,- capabilityClaimToken: 1,- },- },- );- if (shieldResult.modifiedCount === 1) {- return true;- }- const capabilityResult = await Delivery().updateOne(- {- _id: input.id,- ...legacyCapabilityFence(input),- attempts: input.attempt,- },- {- ...update,- $set: { ...update.$set, status: 'capability_pending' },- },- );- if (capabilityResult.modifiedCount === 1) {- return true;- }- const result = await Delivery().updateOne(- { _id: input.id, ...ordinaryFence(input), attempts: input.attempt },- { ...update, $set: { ...update.$set, status: 'pending' } },- );- return result.modifiedCount === 1;+ return releaseWaitingDelivery(input);}async function completeAgentTriggerDelivery(@@ -2287,6 +2263,75 @@ export function createAgentTriggerDeliveryMethods(: { status: 'expired', leaseUntil: delivery.producerLeaseUntil };}+ /** Pulls deferred deliveries back to `now` when the condition they were+ * waiting on has changed, so a waiting delivery can back off without delaying+ * the moment it becomes deliverable. Unclaimed rows move now; held rows retain+ * a signal consumed atomically by readiness deferral or ordering release. */+ async function expediteAgentTriggerDeliveries(+ input: ExpediteAgentTriggerDeliveriesInput,+ ): Promise<ExpediteAgentTriggerDeliveriesResult> {+ const deliveryKeys = input.deliveryKeys ?? [];+ if (+ (input.taskIds != null &&+ (input.user == null ||+ input.conversationId == null ||+ input.taskIds.length === 0 ||+ input.taskIds.some((id) => id.length === 0 || id.length > 256))) ||+ input.sourceIds.length === 0 ||+ input.sourceIds.some((id) => id.length === 0 || id.length > 256) ||+ deliveryKeys.some((key) => key.length === 0 || key.length > 256) ||+ (deliveryKeys.length === 0 && input.user == null) ||+ (input.conversationId != null &&+ (input.conversationId.length === 0 || input.conversationId.length > 256)) ||+ !(input.now instanceof Date) ||+ !Number.isFinite(input.now.getTime())+ ) {+ throw new TypeError('Invalid agent trigger delivery expedite');+ }+ const selection = {+ 'envelope.event.source.type': 'internal',+ 'envelope.event.source.id': { $in: [...input.sourceIds] },+ ...(deliveryKeys.length > 0 && { deliveryKey: { $in: [...deliveryKeys] } }),+ ...(input.taskIds != null && {+ 'envelope.event.payload.taskId': { $in: [...input.taskIds] },+ }),+ ...(input.user != null && { user: input.user }),+ ...(input.conversationId != null && {+ 'envelope.target.conversationId': input.conversationId,+ }),+ };+ /** Classic operators only: aggregation-pipeline updates are not portable. A+ * held row may be deferred on readiness its worker read before the change,+ * so it keeps a marker that its deferral honors instead of moving now. The+ * marker is written first: a row released after it was read as held is then+ * seen unheld by the move, so no release between the two escapes both. */+ const held = await Delivery().updateMany(+ {+ ...selection,+ status: { $in: ['leased', 'capability_leased'] },+ $or: [{ leaseBy: { $exists: true } }, { capabilityLeaseBy: { $exists: true } }],+ },+ { $set: { wakeRequestedAt: input.now } },+ );+ const moved = await Delivery().updateMany(+ {+ ...selection,+ availableAt: { $gt: input.now },+ leaseBy: { $exists: false },+ $or: [+ { status: { $in: ['pending', 'capability_pending'] } },+ {+ status: 'leased',+ capabilityStatus: 'pending',+ capabilityLeaseBy: { $exists: false },+ },+ ],+ },+ { $set: { availableAt: input.now, claimAvailableAt: input.now } },+ );+ return { expedited: moved.modifiedCount, held: held.matchedCount };+ }+/** Stores terminal output on the pre-admitted delivery before attempting the* parent-message projection. The first terminal receipt wins; exact retries* are idempotent and conflicting rewrites fail closed. */@@ -4049,6 +4094,7 @@ export function createAgentTriggerDeliveryMethods(retireAgentTriggerDelivery,renewAgentTriggerDeliveryProducerLease,getAgentTriggerDeliveryProducerLease,+ expediteAgentTriggerDeliveries,persistAgentBackgroundToolResult,getAgentBackgroundToolResult,getAgentBackgroundToolResultClaim,diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.tsindex 2d08f6d14c..d9b503e6c5 100644--- a/packages/data-schemas/src/schema/triggerDelivery.ts+++ b/packages/data-schemas/src/schema/triggerDelivery.ts@@ -218,6 +218,7 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(requeueCount: { type: Number, default: 0, min: 0 },stagingRecoveryAt: { type: Date },laneCleanupPendingAt: { type: Date },+ wakeRequestedAt: { type: Date },},{ timestamps: true },);@@ -253,7 +254,7 @@ triggerDeliverySchema.index({ sparse: true },);triggerDeliverySchema.index({ status: 1, updatedAt: -1 });-/** One user's waiting deliveries, read when their readiness changes. */+/** One user's waiting deliveries, read when that user's generation settles. */triggerDeliverySchema.index({ user: 1, status: 1, availableAt: 1 });triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.tsindex be019d65a2..5865d88267 100644--- a/packages/data-schemas/src/types/triggerDelivery.ts+++ b/packages/data-schemas/src/types/triggerDelivery.ts@@ -171,6 +171,8 @@ export interface IAgentTriggerDelivery {stagingRecoveryAt?: Date;/** Durable proof that successful settlement still owes lane cleanup publication. */laneCleanupPendingAt?: Date;+ /** Readiness changed while a worker held this delivery; its next deferral re-checks at once. */+ wakeRequestedAt?: Date;createdAt?: Date;updatedAt?: Date;}