Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🪂 feat: Make Event Actor Detached Actions Durable (#15307)

Danny Avila487984193a8d
diff --git a/.env.example b/.env.example
index 9b9cea96ed..1c21df2889 100644
--- a/.env.example
+++ b/.env.example
@@ -197,6 +197,10 @@ LOG_TO_FILE=true
# Set to true to enable agent debug logging
AGENT_DEBUG_LOGGING=false
+# Rolling-deployment fence for durable detached Event Actor completions. Requires Redis generation
+# streams. Leave disabled while deploying consumers, then enable after every replica is updated.
+# AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED=false
+
# Enable memory diagnostics (logs heap/RSS snapshots every 60s, auto-enabled with --inspect)
# MEM_DIAG=true
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
index f0bd8e2d03..f8b30502a2 100644
--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
@@ -88,6 +88,7 @@ jest.mock('@librechat/api', () => ({
conversationId,
parentMessageId,
}),
+ parseAgentEventActorDetachedCompletion: jest.fn(() => undefined),
}));
jest.mock('~/server/cleanup', () => ({
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index 2b37fd53c8..d93eb82f68 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -8,6 +8,7 @@ const mockLogger = {
};
const mockGenerationJobManager = {
+ isRedis: false,
createJob: jest.fn(),
getJob: jest.fn(),
emitError: jest.fn(),
@@ -22,6 +23,7 @@ const mockGenerationJobManager = {
failPausePersistence: jest.fn(),
getResumeState: jest.fn(),
updateMetadata: jest.fn(),
+ persistAgentEventDetachedTerminalEvidence: jest.fn(),
claimGeneration: jest.fn(),
resumeClaimedGeneration: jest.fn(),
takeoverGeneration: jest.fn(),
@@ -73,6 +75,8 @@ const mockRecordScheduleOutcome = jest.fn();
const mockIsScheduleLive = jest.fn();
const mockDeleteAgentCheckpoint = jest.fn();
const mockExecuteAgentEventActor = jest.fn();
+const mockResumeAgentEventActor = jest.fn();
+const mockCreateAgentEventActorDetachedActionLifecycle = jest.fn(() => undefined);
const mockFindAgentEventAppliedAction = jest.fn();
const mockResolveAgentTurnExecutionPlan = jest.fn((input) => {
let origin = 'user';
@@ -85,7 +89,6 @@ const mockResolveAgentTurnExecutionPlan = jest.fn((input) => {
input.event?.binding != null &&
input.event?.expectedAction != null &&
input.checkpointerType !== 'memory' &&
- !input.expectedActionMayDetach &&
(!input.canPause || input.durableEventActorSuspensions);
let strategy = 'history';
if (input.isNewConversation) {
@@ -131,6 +134,9 @@ const mockAdmitAgentEventActorAction = jest.fn();
const mockReleaseAgentEventActorAction = jest.fn();
const mockHasAgentEventActorActionAdmission = jest.fn();
const mockGetAgentEventActorReceipt = jest.fn();
+const mockGetAgentEventActorDetachedAction = jest.fn();
+const mockClaimAgentEventActorSuspension = jest.fn();
+const mockSettleAgentEventActorSuspension = jest.fn();
const mockStartupTelemetry = {
mark: jest.fn(),
setStreamId: jest.fn(),
@@ -258,6 +264,13 @@ jest.mock('@librechat/api', () => ({
},
deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
executeAgentEventActor: (...args) => mockExecuteAgentEventActor(...args),
+ resumeAgentEventActor: (...args) => mockResumeAgentEventActor(...args),
+ createAgentEventActorDetachedActionLifecycle: (...args) =>
+ mockCreateAgentEventActorDetachedActionLifecycle(...args),
+ isAgentEventActorDetachedActionProducerEnabled: () => true,
+ parseAgentEventActorDetachedCompletion: (value) => (value?.version === 1 ? value : undefined),
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE: 'librechat-event-actor',
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE: 'librechat.event_actor.detached_completion',
findAgentEventAppliedAction: (...args) => mockFindAgentEventAppliedAction(...args),
createAgentEventActionRecorder: (...args) => mockCreateAgentEventActionRecorder(...args),
resolveAgentTurnExecutionPlan: (...args) => mockResolveAgentTurnExecutionPlan(...args),
@@ -307,6 +320,9 @@ jest.mock('~/models', () => ({
releaseAgentEventActorAction: (...args) => mockReleaseAgentEventActorAction(...args),
hasAgentEventActorActionAdmission: (...args) => mockHasAgentEventActorActionAdmission(...args),
getAgentEventActorReceipt: (...args) => mockGetAgentEventActorReceipt(...args),
+ getAgentEventActorDetachedAction: (...args) => mockGetAgentEventActorDetachedAction(...args),
+ claimAgentEventActorSuspension: (...args) => mockClaimAgentEventActorSuspension(...args),
+ settleAgentEventActorSuspension: (...args) => mockSettleAgentEventActorSuspension(...args),
isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args),
}));
@@ -378,6 +394,8 @@ describe('ResumableAgentController resume metadata', () => {
mockGenerationJobManager.getResumeState.mockResolvedValue(null);
mockGenerationJobManager.getJob.mockResolvedValue(undefined);
mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
+ mockGenerationJobManager.isRedis = false;
+ mockGenerationJobManager.persistAgentEventDetachedTerminalEvidence.mockResolvedValue(true);
mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);
mockGenerationJobManager.emitDone.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
@@ -433,6 +451,9 @@ describe('ResumableAgentController resume metadata', () => {
mockReleaseAgentEventActorAction.mockResolvedValue(true);
mockHasAgentEventActorActionAdmission.mockResolvedValue(false);
mockGetAgentEventActorReceipt.mockResolvedValue(null);
+ mockGetAgentEventActorDetachedAction.mockResolvedValue(null);
+ mockClaimAgentEventActorSuspension.mockResolvedValue({ status: 'claimed' });
+ mockSettleAgentEventActorSuspension.mockResolvedValue({ status: 'settled' });
});
it.each([
@@ -4110,7 +4131,6 @@ describe('ResumableAgentController resume metadata', () => {
conversationId: 'child-conversation',
isNewConversation: false,
canPause: false,
- expectedActionMayDetach: false,
}),
);
expect(mockExecuteAgentEventActor).toHaveBeenCalledWith(
@@ -4120,7 +4140,7 @@ describe('ResumableAgentController resume metadata', () => {
event,
expectedAction: { toolName: 'submit_\u200Fmove', argumentSubset: { expectedPly: 8 } },
}),
- {
+ expect.objectContaining({
getSnapshot: expect.any(Function),
commitState: expect.any(Function),
recordReconciliation: expect.any(Function),
@@ -4130,7 +4150,7 @@ describe('ResumableAgentController resume metadata', () => {
releaseAction: expect.any(Function),
hasActionAdmission: expect.any(Function),
getReceipt: expect.any(Function),
- },
+ }),
);
expect(mockExecuteAgentEventActor.mock.calls[0][0]).not.toHaveProperty('tenantId');
expect(req._agentEventTriggerProjection).toEqual({
@@ -4231,6 +4251,158 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockFindAgentEventAppliedAction).not.toHaveBeenCalled();
});
+ it('resumes the original actor suspension from exact detached terminal evidence', async () => {
+ mockGenerationJobManager.isRedis = true;
+ mockGenerationJobManager.createJob.mockResolvedValue({
+ createdAt: 2000,
+ metadata: {
+ checkpointNamespace: '2000',
+ providerExecutionId: 'provider-segment-2',
+ providerDrained: true,
+ },
+ readyPromise: Promise.resolve(),
+ abortController: new AbortController(),
+ emitter: { on: jest.fn() },
+ });
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(
+ wonGenerationClaim({ streamId: 'child-conversation', conversationId: 'child-conversation' }),
+ );
+ mockGetConvo.mockResolvedValue({
+ conversationId: 'parent-conversation',
+ agent_id: 'parent-agent',
+ tenantId: 'tenant-1',
+ });
+ const suspension = {
+ version: 1,
+ suspensionId: 'suspension-detached-1',
+ attempt: 0,
+ invocation: { invocationId: 'original-delivery-1' },
+ };
+ mockGetAgentEventActorSnapshot.mockResolvedValue({
+ state: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension,
+ actionId: 'task-detached-1',
+ status: 'pending',
+ },
+ });
+ mockGetAgentEventActorDetachedAction.mockResolvedValue({
+ invocationId: 'original-delivery-1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-detached-1',
+ taskId: 'task-detached-1',
+ idempotencyKey: 'a'.repeat(64),
+ status: 'succeeded',
+ result: 'move accepted',
+ });
+ const repaused = { kind: 'internal_completion', actionId: 'task-detached-2' };
+ mockCreateAgentEventActorDetachedActionLifecycle.mockReturnValueOnce({
+ readSuspension: () => repaused,
+ });
+ mockResumeAgentEventActor.mockImplementationOnce(async (input) => {
+ expect(input.readSuspension()).toBe(repaused);
+ throw new Error('stop after resume contract');
+ });
+ const client = { options: { agent: {} }, sendMessage: jest.fn() };
+ const req = {
+ user: { id: 'user-123', tenantId: 'tenant-1' },
+ body: {
+ text: 'Trusted detached completion.',
+ clientRequestId: 'detached-resume-generation-1',
+ conversationId: 'child-conversation',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ agentEventDelivery: {
+ deliveryKey: 'detached-resume-generation-1',
+ target: { bindingId: 'binding-1' },
+ expectedAction: { toolName: 'submit_move' },
+ event: {
+ id: 'task-detached-1',
+ type: 'librechat.event_actor.detached_completion',
+ occurredAt: Date.now(),
+ source: { id: 'librechat-event-actor', type: 'internal' },
+ },
+ internalCompletion: {
+ version: 1,
+ invocationId: 'original-delivery-1',
+ generationCreatedAt: 1000,
+ wakeGenerationCreatedAt: 2000,
+ taskId: 'task-detached-1',
+ idempotencyKey: 'a'.repeat(64),
+ },
+ },
+ },
+ config: { endpoints: { agents: {} } },
+ _isAgentTrigger: true,
+ _agentEventBindingId: 'binding-1',
+ _agentEventBindingParentConversationId: 'parent-conversation',
+ _agentEventBindingParentAgentId: 'parent-agent',
+ _agentEventBindingTenantId: 'tenant-1',
+ _agentEventBindingRetention: { expiredAt: new Date(Date.now() + 60_000) },
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ await nextTick();
+
+ expect(mockExecuteAgentEventActor).not.toHaveBeenCalled();
+ expect(mockResumeAgentEventActor).toHaveBeenCalledWith(
+ expect.objectContaining({
+ conversationId: 'child-conversation',
+ bindingId: 'binding-1',
+ suspension,
+ resumeAttemptId: 'detached-resume-generation-1',
+ resumeValue: expect.objectContaining({
+ taskId: 'task-detached-1',
+ status: 'succeeded',
+ result: 'move accepted',
+ }),
+ }),
+ expect.objectContaining({
+ claimSuspension: expect.any(Function),
+ settleSuspension: expect.any(Function),
+ }),
+ );
+ expect(mockGetAgentEventActorDetachedAction).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'original-delivery-1',
+ generationCreatedAt: 1000,
+ }),
+ );
+ expect(mockCreateAgentEventActorDetachedActionLifecycle).toHaveBeenCalledWith(
+ expect.objectContaining({
+ generationCreatedAt: 1000,
+ turnCreatedAt: 2000,
+ }),
+ expect.any(Object),
+ );
+ const lifecycleDependencies =
+ mockCreateAgentEventActorDetachedActionLifecycle.mock.calls.at(-1)[1];
+ expect(lifecycleDependencies.producerEnabled()).toBe(true);
+ mockGenerationJobManager.isRedis = false;
+ expect(lifecycleDependencies.producerEnabled()).toBe(false);
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'child-conversation',
+ 'user-123',
+ 'child-conversation',
+ expect.objectContaining({
+ initialMetadata: expect.objectContaining({
+ agentEventDeliveryKey: 'detached-resume-generation-1',
+ agentEventInvocationKey: 'original-delivery-1',
+ agentEventInvocationGenerationCreatedAt: 1000,
+ agentEventDetachedActionProducerRequired: true,
+ }),
+ }),
+ );
+ });
+
it('records reconciliation when persistence fails after an event action commits', async () => {
mockGenerationJobManager.claimGeneration.mockResolvedValue(
wonGenerationClaim({ streamId: 'child-conversation', conversationId: 'child-conversation' }),
@@ -4530,9 +4702,7 @@ describe('ResumableAgentController resume metadata', () => {
}),
};
const shouldCheckpoint =
- _label !== 'memory-checkpointer' &&
- _label !== 'background-capable expected action' &&
- _label !== 'pre-cutover pause-capable fleet';
+ _label !== 'memory-checkpointer' && _label !== 'pre-cutover pause-capable fleet';
if (shouldCheckpoint) {
mockExecuteAgentEventActor.mockImplementationOnce(async (input) => {
await input.invoke({
@@ -4671,7 +4841,7 @@ describe('ResumableAgentController resume metadata', () => {
},
},
config: {
- endpoints: { agents: {} },
+ endpoints: { agents: { checkpointer: { type: 'memory' } } },
},
_isAgentTrigger: true,
_agentEventBindingId: 'binding-1',
@@ -4746,7 +4916,7 @@ describe('ResumableAgentController resume metadata', () => {
},
},
config: {
- endpoints: { agents: {} },
+ endpoints: { agents: { checkpointer: { type: 'memory' } } },
},
_isAgentTrigger: true,
_agentEventBindingId: 'binding-1',
@@ -4817,7 +4987,7 @@ describe('ResumableAgentController resume metadata', () => {
},
},
config: {
- endpoints: { agents: {} },
+ endpoints: { agents: { checkpointer: { type: 'memory' } } },
},
_isAgentTrigger: true,
_agentEventBindingId: 'binding-1',
@@ -4899,7 +5069,7 @@ describe('ResumableAgentController resume metadata', () => {
},
},
config: {
- endpoints: { agents: {} },
+ endpoints: { agents: { checkpointer: { type: 'memory' } } },
},
_isAgentTrigger: true,
_agentEventBindingId: 'binding-1',
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js
index 8b73032b25..3d8646ccec 100644
--- a/api/server/controllers/agents/__tests__/resume.spec.js
+++ b/api/server/controllers/agents/__tests__/resume.spec.js
@@ -50,6 +50,7 @@ const mockGenerationJobManager = {
getJob: jest.fn(),
getJobStore: jest.fn(() => mockJobStore),
getResumeState: jest.fn(),
+ updateMetadata: jest.fn(),
setContentParts: jest.fn(),
/** Resume moves ownership and rebuilds armed interrupts from the durable queue. */
rearmQueuedPreempts: jest.fn().mockResolvedValue(0),
@@ -110,6 +111,12 @@ const mockClaimAgentEventActorSuspension = jest.fn();
const mockSettleAgentEventActorSuspension = jest.fn();
const mockRecordAgentEventActorReconciliation = jest.fn();
const mockResumeAgentEventActor = jest.fn();
+const mockCreateAgentEventActorDetachedActionLifecycle = jest.fn();
+const mockIsAgentEventActorDetachedActionProducerEnabled = jest.fn();
+const mockGetAgentTriggerDelivery = jest.fn();
+const mockReserveAgentEventActorDetachedAction = jest.fn();
+const mockMarkAgentEventActorDetachedActionRunning = jest.fn();
+const mockSettleAgentEventActorDetachedAction = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
@@ -132,6 +139,10 @@ jest.mock('@librechat/api', () => ({
getAgentCheckpointer: (...args) => mockGetAgentCheckpointer(...args),
checkAccess: (...args) => mockCheckAccess(...args),
resumeAgentEventActor: (...args) => mockResumeAgentEventActor(...args),
+ createAgentEventActorDetachedActionLifecycle: (...args) =>
+ mockCreateAgentEventActorDetachedActionLifecycle(...args),
+ isAgentEventActorDetachedActionProducerEnabled: () =>
+ mockIsAgentEventActorDetachedActionProducerEnabled(),
}));
jest.mock('~/models', () => ({
@@ -143,6 +154,7 @@ jest.mock('~/models', () => ({
getActions: (...args) => mockGetActions(...args),
getUserMemories: (...args) => mockGetUserMemories(...args),
getRoleByName: (...args) => mockGetRoleByName(...args),
+ getAgentTriggerDelivery: (...args) => mockGetAgentTriggerDelivery(...args),
isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args),
completeAgentEventActorLegacyTurn: (...args) => mockCompleteAgentEventActorLegacyTurn(...args),
getAgentEventActorSnapshot: (...args) => mockGetAgentEventActorSnapshot(...args),
@@ -152,6 +164,12 @@ jest.mock('~/models', () => ({
settleAgentEventActorSuspension: (...args) => mockSettleAgentEventActorSuspension(...args),
recordAgentEventActorReconciliation: (...args) =>
mockRecordAgentEventActorReconciliation(...args),
+ reserveAgentEventActorDetachedAction: (...args) =>
+ mockReserveAgentEventActorDetachedAction(...args),
+ markAgentEventActorDetachedActionRunning: (...args) =>
+ mockMarkAgentEventActorDetachedActionRunning(...args),
+ settleAgentEventActorDetachedAction: (...args) =>
+ mockSettleAgentEventActorDetachedAction(...args),
}));
jest.mock('~/server/services/Endpoints/agents/eventChildLease', () => ({
@@ -325,6 +343,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
});
mockJobStore.updateJob.mockResolvedValue(undefined);
mockGenerationJobManager.getResumeState.mockResolvedValue({ aggregatedContent: [] });
+ mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
mockGenerationJobManager.emitDone.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);
@@ -352,6 +371,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
+ mockGenerationJobManager.isRedis = true;
+ mockGenerationJobManager.persistAgentEventDetachedTerminalEvidence = jest
+ .fn()
+ .mockResolvedValue(true);
+ mockGenerationJobManager.retryTerminalHostAction = jest.fn().mockResolvedValue(undefined);
mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);
mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);
@@ -375,6 +399,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockClaimAgentEventActorSuspension.mockResolvedValue({ status: 'claimed' });
mockSettleAgentEventActorSuspension.mockResolvedValue({ status: 'settled' });
mockRecordAgentEventActorReconciliation.mockResolvedValue(true);
+ mockIsAgentEventActorDetachedActionProducerEnabled.mockReturnValue(true);
+ mockGetAgentTriggerDelivery.mockResolvedValue(undefined);
+ mockCreateAgentEventActorDetachedActionLifecycle.mockReturnValue(undefined);
endpointAgent = {
_id: 'mongo-agent-abc',
id: AGENT_ID,
@@ -501,6 +528,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
const pausedJob = makeToolApprovalJob({
metadata: {
idempotencyClientRequestId: 'trigger_event_delivery',
+ agentEventDeliveryKey: 'completion-delivery-1',
+ agentEventInvocationKey: 'trigger_event_delivery',
agentEventExpectedAction: expectedAction,
agentEventSuspension: {
version: 1,
@@ -513,6 +542,18 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
{ tool_call_id: 'tc1', allowed_decisions: ['respond'] },
];
mockGenerationJobManager.getJob.mockResolvedValue(pausedJob);
+ const detachedSuspension = {
+ kind: 'internal_completion',
+ actionId: 'detached-action-next',
+ };
+ const repausedSuspension = {
+ ...suspension,
+ suspensionId: 'suspension-detached-next',
+ attempt: 1,
+ };
+ mockCreateAgentEventActorDetachedActionLifecycle.mockReturnValue({
+ readSuspension: () => detachedSuspension,
+ });
let projectedProviderExecutionId;
mockGenerationJobManager.approvals.resolve.mockImplementation(
async (_streamId, _actionId, resumePatch) => {
@@ -535,12 +576,15 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
reconciliations: [],
suspension: {
suspension,
+ kind: 'internal_completion',
+ handlingGenerationCreatedAt: 500,
actionId: ACTION_ID,
jobCreatedAt: 1000,
status: 'pending',
},
});
const resumedClient = makeClient({
+ publishStagedApproval: jest.fn(),
contentParts: [makeToolCallContent({ output: 'human supplied output' })],
run: {
getRunSteps: () => [
@@ -562,8 +606,14 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
],
},
});
- mockInitializeClient.mockResolvedValue({ client: resumedClient, userMCPAuthMap: {} });
+ mockInitializeClient.mockImplementation(async ({ req }) => {
+ expect(req._agentEventDetachedActionLifecycle).toBe(
+ mockCreateAgentEventActorDetachedActionLifecycle.mock.results[0].value,
+ );
+ return { client: resumedClient, userMCPAuthMap: {} };
+ });
mockResumeAgentEventActor.mockImplementation(async (input, dependencies) => {
+ expect(input.readSuspension()).toBe(detachedSuspension);
await dependencies.claimSuspension({
user: USER_ID,
tenantId: TENANT_ID,
@@ -583,10 +633,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
continuation: 'warm',
signal: input.signal,
});
- expect(input.readAppliedAction()).toBeUndefined();
return {
value,
- execution: { status: 'completed_no_action' },
+ execution: { status: 'suspended', suspension: repausedSuspension },
};
});
@@ -610,40 +659,60 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0],
);
expect(mockGenerationJobManager.getJob).toHaveBeenCalledTimes(2);
+ expect(mockGetAgentTriggerDelivery).not.toHaveBeenCalled();
+ expect(mockCreateAgentEventActorDetachedActionLifecycle).toHaveBeenCalledWith(
+ expect.objectContaining({
+ invocationId: 'trigger_event_delivery',
+ generationCreatedAt: 500,
+ turnCreatedAt: 1000,
+ expectedAction,
+ }),
+ expect.objectContaining({
+ reserveAgentEventActorDetachedAction: expect.any(Function),
+ markAgentEventActorDetachedActionRunning: expect.any(Function),
+ settleAgentEventActorDetachedAction: expect.any(Function),
+ }),
+ );
+ expect(
+ mockCreateAgentEventActorDetachedActionLifecycle.mock.invocationCallOrder[0],
+ ).toBeLessThan(mockInitializeClient.mock.invocationCallOrder[0]);
+ const lifecycleDependencies =
+ mockCreateAgentEventActorDetachedActionLifecycle.mock.calls[0][1];
+ expect(lifecycleDependencies.producerEnabled()).toBe(true);
+ mockGenerationJobManager.isRedis = false;
+ expect(lifecycleDependencies.producerEnabled()).toBe(false);
expect(resumedClient.resumeCompletion).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.updateMetadata).toHaveBeenCalledWith(
+ CONVO_ID,
+ {
+ agentEventSuspension: {
+ version: repausedSuspension.version,
+ suspensionId: repausedSuspension.suspensionId,
+ attempt: repausedSuspension.attempt,
+ },
+ },
+ 1000,
+ );
+ expect(resumedClient.publishStagedApproval).not.toHaveBeenCalled();
expect(mockRecordAgentEventActorReconciliation).not.toHaveBeenCalled();
});
- it('does not record provider execution when client reconstruction fails before continuation', async () => {
+ it('refuses a prior-head expected-action suspension before claiming it on an incapable replica', async () => {
configureEventActorResume();
+ requestStateOverrides._agentEventBindingId = 'binding-1';
+ const expectedAction = { toolName: 'submit_move' };
const suspension = {
version: 1,
- suspensionId: 'suspension-init-failure',
+ suspensionId: 'suspension-producer-required',
attempt: 0,
issuedAt: Date.now(),
expiresAt: Date.now() + 60_000,
- invocation: {
- invocationId: 'trigger_event_delivery',
- continuation: 'warm',
- base: { actorThreadId: CONVO_ID, generation: 0 },
- fork: {
- threadId: CONVO_ID,
- checkpointNs: 'event-actor',
- checkpointId: 'checkpoint-paused',
- invocationId: 'trigger_event_delivery',
- },
- },
- checkpoint: {
- threadId: CONVO_ID,
- checkpointNs: 'event-actor',
- checkpointId: 'checkpoint-paused',
- invocationId: 'trigger_event_delivery',
- },
+ invocation: { invocationId: 'trigger_event_delivery' },
interrupt: {
- id: 'interrupt-init-failure',
+ id: 'interrupt-producer-required',
payload: {
type: 'tool_approval',
- _librechatEventActor: { expectedAction: { toolName: 'lookup' } },
+ _librechatEventActor: { expectedAction },
},
},
suspensionDigest: 'signed-digest',
@@ -652,6 +721,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
makeToolApprovalJob({
metadata: {
idempotencyClientRequestId: 'trigger_event_delivery',
+ agentEventDeliveryKey: 'trigger_event_delivery',
+ agentEventExpectedAction: expectedAction,
agentEventSuspension: {
version: 1,
suspensionId: suspension.suspensionId,
@@ -667,33 +738,119 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
reconciliations: [],
suspension: {
suspension,
+ kind: 'human_decision',
+ handlingGenerationCreatedAt: 1000,
actionId: ACTION_ID,
jobCreatedAt: 1000,
status: 'pending',
},
});
- mockInitializeClient.mockRejectedValue(new Error('client reconstruction failed'));
- mockResumeAgentEventActor.mockImplementation(async (input) => {
- expect(await input.claimProjection()).toBe(true);
- return input.resume({
- checkpointNamespace: 'event-actor',
- checkpointId: 'checkpoint-paused',
- invocationId: 'trigger_event_delivery',
- continuation: 'warm',
- signal: input.signal,
- });
- });
+ mockIsAgentEventActorDetachedActionProducerEnabled.mockReturnValue(false);
const res = await post(approveBody());
- expect(res.status).toBe(200);
- await settled;
- await flush();
- expect(mockInitializeClient).toHaveBeenCalledTimes(1);
- expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();
+ expect(res.status).toBe(503);
+ expect(res.body).toMatchObject({
+ code: 'EVENT_ACTOR_RESUME_CAPABILITY_UNAVAILABLE',
+ });
+ expect(res.headers['retry-after']).toBe('1');
+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
+ expect(mockResumeAgentEventActor).not.toHaveBeenCalled();
+ expect(mockInitializeClient).not.toHaveBeenCalled();
+ expect(mockDecrementPendingRequest).toHaveBeenCalledTimes(1);
+ expect(mockReleaseEventChildLease).toHaveBeenCalledTimes(1);
});
+ it.each([
+ ['pre-feature expected-action', { toolName: 'lookup' }, undefined],
+ ['actionless PR-era', undefined, 1000],
+ ])(
+ 'preserves a %s suspension when client reconstruction fails before continuation',
+ async (_case, expectedAction, handlingGenerationCreatedAt) => {
+ configureEventActorResume();
+ const suspension = {
+ version: 1,
+ suspensionId: 'suspension-init-failure',
+ attempt: 0,
+ issuedAt: Date.now(),
+ expiresAt: Date.now() + 60_000,
+ invocation: {
+ invocationId: 'trigger_event_delivery',
+ continuation: 'warm',
+ base: { actorThreadId: CONVO_ID, generation: 0 },
+ fork: {
+ threadId: CONVO_ID,
+ checkpointNs: 'event-actor',
+ checkpointId: 'checkpoint-paused',
+ invocationId: 'trigger_event_delivery',
+ },
+ },
+ checkpoint: {
+ threadId: CONVO_ID,
+ checkpointNs: 'event-actor',
+ checkpointId: 'checkpoint-paused',
+ invocationId: 'trigger_event_delivery',
+ },
+ interrupt: {
+ id: 'interrupt-init-failure',
+ payload: {
+ type: 'tool_approval',
+ _librechatEventActor: expectedAction == null ? {} : { expectedAction },
+ },
+ },
+ suspensionDigest: 'signed-digest',
+ };
+ mockGenerationJobManager.getJob.mockResolvedValue(
+ makeToolApprovalJob({
+ metadata: {
+ idempotencyClientRequestId: 'trigger_event_delivery',
+ ...(expectedAction == null ? {} : { agentEventExpectedAction: expectedAction }),
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: suspension.attempt,
+ },
+ },
+ }),
+ );
+ mockGetAgentEventActorSnapshot.mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ suspension,
+ ...(handlingGenerationCreatedAt == null ? {} : { handlingGenerationCreatedAt }),
+ actionId: ACTION_ID,
+ jobCreatedAt: 1000,
+ status: 'pending',
+ },
+ });
+ mockIsAgentEventActorDetachedActionProducerEnabled.mockReturnValue(false);
+ mockInitializeClient.mockRejectedValue(new Error('client reconstruction failed'));
+ mockResumeAgentEventActor.mockImplementation(async (input) => {
+ expect(await input.claimProjection()).toBe(true);
+ return input.resume({
+ checkpointNamespace: 'event-actor',
+ checkpointId: 'checkpoint-paused',
+ invocationId: 'trigger_event_delivery',
+ continuation: 'warm',
+ signal: input.signal,
+ });
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockInitializeClient).toHaveBeenCalledTimes(1);
+ expect(mockCreateAgentEventActorDetachedActionLifecycle).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();
+ },
+ );
+
it('fails closed when a versioned job marker no longer matches canonical suspension', async () => {
configureEventActorResume();
mockGenerationJobManager.getJob.mockResolvedValue(
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index 87ddfbde76..7cc2613c5c 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -31,6 +31,12 @@ const {
createMCPRuntimeRequestBody,
isAgentEventRetentionActive,
executeAgentEventActor,
+ resumeAgentEventActor,
+ createAgentEventActorDetachedActionLifecycle,
+ isAgentEventActorDetachedActionProducerEnabled,
+ parseAgentEventActorDetachedCompletion,
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
findAgentEventAppliedAction,
createAgentEventActionRecorder,
isHITLEnabled,
@@ -61,6 +67,12 @@ const {
releaseAgentEventActorAction,
hasAgentEventActorActionAdmission,
getAgentEventActorReceipt,
+ getAgentEventActorDetachedAction,
+ reserveAgentEventActorDetachedAction,
+ markAgentEventActorDetachedActionRunning,
+ settleAgentEventActorDetachedAction,
+ claimAgentEventActorSuspension,
+ settleAgentEventActorSuspension,
isAgentTriggerPrincipalActive,
isSubagentOwnerAdmissible,
} = require('~/models');
@@ -1310,12 +1322,21 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
* idempotency key that owns generation admission. Ignore mismatched or
* direct-chat metadata rather than letting callers relabel another run. */
const rawAgentEventDelivery = req.body?.agentEventDelivery;
+ const internalDetachedCompletion = parseAgentEventActorDetachedCompletion(
+ rawAgentEventDelivery?.internalCompletion,
+ );
+ const isInternalDetachedCompletion =
+ internalDetachedCompletion != null &&
+ rawAgentEventDelivery?.deliveryKey === clientRequestId &&
+ rawAgentEventDelivery?.event?.type === EVENT_ACTOR_DETACHED_COMPLETION_TYPE &&
+ rawAgentEventDelivery?.event?.source?.type === 'internal' &&
+ rawAgentEventDelivery?.event?.source?.id === EVENT_ACTOR_DETACHED_COMPLETION_SOURCE;
const agentEventDelivery =
isTriggerContinuation &&
isBoundEventContinuation &&
rawAgentEventDelivery != null &&
typeof rawAgentEventDelivery === 'object' &&
- rawAgentEventDelivery.deliveryKey === clientRequestId
+ (rawAgentEventDelivery.deliveryKey === clientRequestId || isInternalDetachedCompletion)
? rawAgentEventDelivery
: undefined;
req._agentEventTriggerProjection = getAgentEventTriggerProjection(agentEventDelivery);
@@ -1367,9 +1388,20 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
isTemporary: req._agentEventBindingRetention?.isTemporary ?? req.body?.isTemporary,
...(agentEventDelivery != null && {
agentEventDeliveryKey: agentEventDelivery.deliveryKey,
+ ...(internalDetachedCompletion == null
+ ? {}
+ : {
+ agentEventInvocationKey: internalDetachedCompletion.invocationId,
+ agentEventInvocationGenerationCreatedAt:
+ internalDetachedCompletion.generationCreatedAt,
+ }),
agentEventBindingId: boundEventBindingId,
...(agentEventDelivery.expectedAction != null && {
agentEventExpectedAction: agentEventDelivery.expectedAction,
+ ...(GenerationJobManager.isRedis &&
+ isAgentEventActorDetachedActionProducerEnabled() && {
+ agentEventDetachedActionProducerRequired: true,
+ }),
}),
}),
...(isRegenerate && { isRegenerate: true }),
@@ -1691,15 +1723,6 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
const eventActorMayPause =
isHITLEnabled(agentsConfig?.toolApproval) ||
eventActorAgents.some(agentRequestsAskUserQuestion);
- const expectedActionToolName = agentEventDelivery?.expectedAction?.toolName;
- const eventActorActionMayDetach =
- typeof expectedActionToolName === 'string' &&
- eventActorAgents.some((agent) =>
- (agent.backgroundToolNames ?? []).some(
- (name) =>
- name === expectedActionToolName || name.startsWith(`${expectedActionToolName}_mcp_`),
- ),
- );
const turnExecutionPlan = resolveAgentTurnExecutionPlan({
conversationId,
parentMessageId,
@@ -1731,7 +1754,6 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
* `/resume` replica from consuming a signed suspension it cannot claim. */
durableEventActorSuspensions: generationProtocolVersion >= GENERATION_PROTOCOL_V2,
checkpointerType: agentsConfig?.checkpointer?.type,
- expectedActionMayDetach: eventActorActionMayDetach,
});
// Resolve title timing from the public agents endpoint first, then fall
@@ -2061,67 +2083,213 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
const eventActorActionRecorder = usesCheckpointStrategy
? createAgentEventActionRecorder(turnExecutionPlan.expectedAction)
: undefined;
+ const actorInvocationId = internalDetachedCompletion?.invocationId ?? eventTaskId;
+ const eventActorDetachedAction =
+ usesCheckpointStrategy &&
+ turnExecutionPlan.expectedAction != null &&
+ turnExecutionPlan.binding != null
+ ? createAgentEventActorDetachedActionLifecycle(
+ {
+ user: userId,
+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),
+ bindingId: turnExecutionPlan.binding.bindingId,
+ conversationId,
+ generationCreatedAt:
+ internalDetachedCompletion?.generationCreatedAt ?? jobCreatedAt,
+ turnCreatedAt: jobCreatedAt,
+ invocationId: actorInvocationId,
+ expectedAction: turnExecutionPlan.expectedAction,
+ },
+ {
+ reserveAgentEventActorDetachedAction,
+ markAgentEventActorDetachedActionRunning,
+ settleAgentEventActorDetachedAction,
+ producerEnabled: () =>
+ GenerationJobManager.isRedis &&
+ isAgentEventActorDetachedActionProducerEnabled(),
+ persistTerminalEvidence: async (evidence) => {
+ const persisted =
+ await GenerationJobManager.persistAgentEventDetachedTerminalEvidence(
+ streamId,
+ jobCreatedAt,
+ evidence,
+ );
+ if (!persisted) {
+ throw new Error(
+ 'Detached Event Actor terminal retry evidence could not be staged',
+ );
+ }
+ },
+ /** Retry immediately when the generation already reached its
+ * terminal host-action fence. The same durable marker is
+ * recovered across replicas and restarts by the existing
+ * GenerationJobManager sweep. */
+ onTerminal: async () => {
+ await GenerationJobManager.retryTerminalHostAction(streamId, jobCreatedAt);
+ },
+ },
+ )
+ : undefined;
+ req._agentEventDetachedActionLifecycle = eventActorDetachedAction;
if (eventActorActionRecorder != null) {
req._agentEventActionObserver = eventActorActionRecorder.observeToolEnd;
}
+ let internalDetachedAction;
+ let internalDetachedSuspension;
+ if (usesCheckpointStrategy && isInternalDetachedCompletion) {
+ const snapshot = await getAgentEventActorSnapshot({
+ user: userId,
+ conversationId,
+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),
+ });
+ internalDetachedSuspension = snapshot?.suspension;
+ internalDetachedAction = await getAgentEventActorDetachedAction({
+ deliveryKey: internalDetachedCompletion.invocationId,
+ user: userId,
+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),
+ bindingId: turnExecutionPlan.binding.bindingId,
+ conversationId,
+ generationCreatedAt: internalDetachedCompletion.generationCreatedAt,
+ });
+ if (
+ internalDetachedSuspension?.kind !== 'internal_completion' ||
+ internalDetachedSuspension.status !== 'pending' ||
+ internalDetachedSuspension.actionId !== internalDetachedCompletion.taskId ||
+ internalDetachedSuspension.suspension.invocation.invocationId !==
+ internalDetachedCompletion.invocationId ||
+ internalDetachedAction == null ||
+ internalDetachedAction.taskId !== internalDetachedCompletion.taskId ||
+ internalDetachedAction.idempotencyKey !== internalDetachedCompletion.idempotencyKey ||
+ !['succeeded', 'failed', 'cancelled'].includes(internalDetachedAction.status)
+ ) {
+ throw Object.assign(
+ new Error('The detached Event Actor completion is no longer current'),
+ { code: 'EVENT_ACTOR_NOT_READY', status: 409 },
+ );
+ }
+ }
+ const readAppliedEventAction = () =>
+ eventActorActionRecorder.read() ??
+ (internalDetachedAction?.status === 'succeeded'
+ ? {
+ toolName: internalDetachedAction.toolName,
+ toolCallId: internalDetachedAction.toolCallId,
+ }
+ : undefined) ??
+ findAgentEventAppliedAction(
+ turnExecutionPlan.expectedAction,
+ client?.run?.getRunSteps?.() ?? [],
+ client?.contentParts ?? [],
+ );
+ const actorDependencies = {
+ getSnapshot: getAgentEventActorSnapshot,
+ commitState: commitAgentEventActorState,
+ storeSuspension: storeAgentEventActorSuspension,
+ claimSuspension: claimAgentEventActorSuspension,
+ settleSuspension: settleAgentEventActorSuspension,
+ recordReconciliation: recordAgentEventActorReconciliation,
+ resolveReconciliation: resolveAgentEventActorReconciliation,
+ admitAction: admitAgentEventActorAction,
+ releaseAction: releaseAgentEventActorAction,
+ hasActionAdmission: hasAgentEventActorActionAdmission,
+ getReceipt: getAgentEventActorReceipt,
+ clearReconciliation: clearAgentEventActorReconciliation,
+ };
const sendPromise = usesCheckpointStrategy
- ? executeAgentEventActor(
- {
- user: userId,
- ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),
- conversationId,
- bindingId: turnExecutionPlan.binding.bindingId,
- invocationId: eventTaskId,
- event: agentEventDelivery.event,
- expectedAction: turnExecutionPlan.expectedAction,
- signal: job.abortController.signal,
- checkpointer: req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
- resolveContext: (state) => client.prepareEventActorContext(state),
- readResultContext: () => client.getEventActorContext(),
- invoke: async (actorContext) => {
- client.checkpointNamespace = actorContext.checkpointNamespace;
- client.eventActorCheckpointId = actorContext.checkpointId;
- client.eventActorInvocationId = actorContext.invocationId;
- client.eventActorContinuation = actorContext.continuation;
- return client.sendMessage(text, messageOptions);
- },
- readAppliedAction: () =>
- eventActorActionRecorder.read() ??
- findAgentEventAppliedAction(
- turnExecutionPlan.expectedAction,
- client?.run?.getRunSteps?.() ?? [],
- client?.contentParts ?? [],
- ),
- readSuspension: () => client.readEventActorSuspension(),
- },
- {
- getSnapshot: getAgentEventActorSnapshot,
- commitState: commitAgentEventActorState,
- storeSuspension: storeAgentEventActorSuspension,
- recordReconciliation: recordAgentEventActorReconciliation,
- resolveReconciliation: resolveAgentEventActorReconciliation,
- admitAction: admitAgentEventActorAction,
- releaseAction: releaseAgentEventActorAction,
- hasActionAdmission: hasAgentEventActorActionAdmission,
- getReceipt: getAgentEventActorReceipt,
- clearReconciliation: clearAgentEventActorReconciliation,
- },
+ ? (isInternalDetachedCompletion
+ ? resumeAgentEventActor(
+ {
+ user: userId,
+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),
+ conversationId,
+ bindingId: turnExecutionPlan.binding.bindingId,
+ suspension: internalDetachedSuspension.suspension,
+ resumeAttemptId: clientRequestId,
+ resumeValue: {
+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ taskId: internalDetachedAction.taskId,
+ status: internalDetachedAction.status,
+ ...(internalDetachedAction.result == null
+ ? {}
+ : { result: internalDetachedAction.result }),
+ ...(internalDetachedAction.error == null
+ ? {}
+ : { error: internalDetachedAction.error }),
+ },
+ signal: job.abortController.signal,
+ checkpointer: req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
+ expectedAction: turnExecutionPlan.expectedAction,
+ resume: async (actorContext) => {
+ client.checkpointNamespace = actorContext.checkpointNamespace;
+ client.eventActorCheckpointId = actorContext.checkpointId;
+ client.eventActorInvocationId = actorContext.invocationId;
+ client.eventActorContinuation = actorContext.continuation;
+ return client.sendMessage(text, messageOptions);
+ },
+ readAppliedAction: readAppliedEventAction,
+ readSuspension: () =>
+ eventActorDetachedAction?.readSuspension() ??
+ client.readEventActorSuspension(),
+ readResultContext: () => client.getEventActorContext(),
+ },
+ actorDependencies,
+ )
+ : executeAgentEventActor(
+ {
+ user: userId,
+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),
+ conversationId,
+ bindingId: turnExecutionPlan.binding.bindingId,
+ invocationId: actorInvocationId,
+ event: agentEventDelivery.event,
+ expectedAction: turnExecutionPlan.expectedAction,
+ signal: job.abortController.signal,
+ checkpointer: req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
+ resolveContext: (state) => client.prepareEventActorContext(state),
+ readResultContext: () => client.getEventActorContext(),
+ invoke: async (actorContext) => {
+ client.checkpointNamespace = actorContext.checkpointNamespace;
+ client.eventActorCheckpointId = actorContext.checkpointId;
+ client.eventActorInvocationId = actorContext.invocationId;
+ client.eventActorContinuation = actorContext.continuation;
+ return client.sendMessage(text, messageOptions);
+ },
+ readAppliedAction: readAppliedEventAction,
+ readSuspension: () =>
+ eventActorDetachedAction?.readSuspension() ??
+ client.readEventActorSuspension(),
+ },
+ actorDependencies,
+ )
).then(async ({ value, execution }) => {
if (execution.status === 'applied') {
appliedEventActor = {
- invocationId: eventTaskId,
+ invocationId: actorInvocationId,
actionAdmitted: typeof admitAgentEventActorAction === 'function',
checkpoint: execution.head.checkpoint,
action: execution.result.action,
};
} else if (execution.status === 'suspended') {
- if (!(await client.publishStagedApproval(execution.suspension))) {
+ const suspensionKind = eventActorDetachedAction?.readSuspension()?.kind;
+ if (suspensionKind === 'internal_completion') {
+ await GenerationJobManager.updateMetadata(
+ streamId,
+ {
+ agentEventSuspension: {
+ version: execution.suspension.version,
+ suspensionId: execution.suspension.suspensionId,
+ attempt: execution.suspension.attempt,
+ },
+ },
+ jobCreatedAt,
+ );
+ } else if (!(await client.publishStagedApproval(execution.suspension))) {
throw new Error('Event actor suspension could not be projected to its job');
}
}
logger.info('[event-actor] Bound child event completed', {
conversationId,
- invocationId: eventTaskId,
+ invocationId: actorInvocationId,
status: execution.status,
continuation: execution.continuation,
});
diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js
index f6644de811..c7fe2d697e 100644
--- a/api/server/controllers/agents/resume.js
+++ b/api/server/controllers/agents/resume.js
@@ -42,6 +42,8 @@ const {
isAgentEventRetentionActive,
resumeAgentEventActor,
createAgentEventActionRecorder,
+ createAgentEventActorDetachedActionLifecycle,
+ isAgentEventActorDetachedActionProducerEnabled,
findAgentEventAppliedAction,
} = require('@librechat/api');
const { disposeClient } = require('~/server/cleanup');
@@ -60,6 +62,7 @@ const {
getActions,
getUserMemories,
getRoleByName,
+ getAgentTriggerDelivery,
isSubagentOwnerAdmissible,
getAgentEventActorSnapshot,
commitAgentEventActorState,
@@ -68,6 +71,9 @@ const {
settleAgentEventActorSuspension,
recordAgentEventActorReconciliation,
completeAgentEventActorLegacyTurn,
+ reserveAgentEventActorDetachedAction,
+ markAgentEventActorDetachedActionRunning,
+ settleAgentEventActorDetachedAction,
} = require('~/models');
const {
acquireEventChildGenerationLease,
@@ -1196,6 +1202,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
let releaseEventChildLease;
let eventLeaseTransferredToRun = false;
let durableEventActorSuspension;
+ let durableEventActorHandlingGenerationCreatedAt;
+ let durableEventActorRequiresDetachedProducer = false;
let eventActorResumePromise;
let eventActorStartGate;
let eventActorContinuationStarted = false;
@@ -1330,6 +1338,14 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
suspensionRecord.suspension.attempt === suspensionProjection.attempt
) {
durableEventActorSuspension = suspensionRecord.suspension;
+ durableEventActorHandlingGenerationCreatedAt =
+ suspensionRecord.handlingGenerationCreatedAt;
+ durableEventActorRequiresDetachedProducer =
+ job.metadata.agentEventDetachedActionProducerRequired === true ||
+ (suspensionRecord.handlingGenerationCreatedAt != null &&
+ job.metadata.agentEventExpectedAction != null) ||
+ job.metadata.agentEventInvocationKey != null ||
+ suspensionRecord.kind === 'internal_completion';
const signedExpectedAction = getSuspendedEventActorExpectedAction(
durableEventActorSuspension,
);
@@ -1367,6 +1383,26 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
);
}
}
+ if (
+ durableEventActorSuspension != null &&
+ durableEventActorRequiresDetachedProducer &&
+ (!GenerationJobManager.isRedis || !isAgentEventActorDetachedActionProducerEnabled())
+ ) {
+ const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);
+ await rollbackUnconsumedScheduleClaim(currentJob);
+ await releaseScheduleFence();
+ await decrementPendingRequest(userId);
+ res.set('Retry-After', '1');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'EVENT_ACTOR_RESUME_CAPABILITY_UNAVAILABLE',
+ error: 'A durable Event Actor resume worker is temporarily unavailable',
+ },
+ generationProtocolVersion,
+ );
+ }
}
// Atomically claim the resume. The single winner drives the run; a racing second
@@ -1401,6 +1437,65 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
const claimGate = deferred();
eventActorStartGate = deferred();
const expectedAction = getSuspendedEventActorExpectedAction(durableEventActorSuspension);
+ const actorInvocationId =
+ job.metadata.agentEventInvocationKey ?? job.metadata.agentEventDeliveryKey;
+ let actorInvocationGenerationCreatedAt =
+ job.metadata.agentEventInvocationGenerationCreatedAt ??
+ durableEventActorHandlingGenerationCreatedAt ??
+ (job.metadata.agentEventInvocationKey == null ? job.createdAt : undefined);
+ if (
+ actorInvocationGenerationCreatedAt == null &&
+ job.metadata.agentEventInvocationKey != null
+ ) {
+ const originalDelivery = await getAgentTriggerDelivery(
+ job.metadata.agentEventInvocationKey,
+ );
+ actorInvocationGenerationCreatedAt = originalDelivery?.handling?.generationCreatedAt;
+ }
+ if (
+ durableEventActorRequiresDetachedProducer &&
+ actorInvocationId != null &&
+ Number.isSafeInteger(actorInvocationGenerationCreatedAt) &&
+ req._agentEventBindingId != null
+ ) {
+ req._agentEventDetachedActionLifecycle = createAgentEventActorDetachedActionLifecycle(
+ {
+ user: userId,
+ ...(req._agentEventBindingTenantId == null
+ ? {}
+ : { tenantId: req._agentEventBindingTenantId }),
+ bindingId: req._agentEventBindingId,
+ conversationId,
+ generationCreatedAt: actorInvocationGenerationCreatedAt,
+ turnCreatedAt: job.createdAt,
+ invocationId: actorInvocationId,
+ expectedAction,
+ },
+ {
+ reserveAgentEventActorDetachedAction,
+ markAgentEventActorDetachedActionRunning,
+ settleAgentEventActorDetachedAction,
+ producerEnabled: () =>
+ GenerationJobManager.isRedis && isAgentEventActorDetachedActionProducerEnabled(),
+ persistTerminalEvidence: async (evidence) => {
+ const persisted =
+ await GenerationJobManager.persistAgentEventDetachedTerminalEvidence(
+ streamId,
+ job.createdAt,
+ evidence,
+ );
+ if (!persisted) {
+ throw new Error(
+ 'Detached Event Actor terminal retry evidence could not be staged',
+ );
+ }
+ },
+ onTerminal: async () => {
+ await GenerationJobManager.retryTerminalHostAction(streamId, job.createdAt);
+ },
+ },
+ );
+ }
eventActorActionRecorder = createAgentEventActionRecorder(expectedAction);
req._agentEventActionObserver = eventActorActionRecorder.observeToolEnd;
eventActorResumePromise = resumeAgentEventActor(
@@ -1454,7 +1549,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
client?.contentParts ?? [],
{ userSubmittedMessageFieldPaths },
),
- readSuspension: () => client?.readEventActorSuspension(),
+ readSuspension: () =>
+ req._agentEventDetachedActionLifecycle?.readSuspension() ??
+ client?.readEventActorSuspension(),
readResultContext: () => client?.getEventActorContext(),
},
{
@@ -1749,7 +1846,20 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
});
const actorResult = await eventActorResumePromise;
if (actorResult.execution.status === 'suspended') {
- if (!(await client.publishStagedApproval(actorResult.execution.suspension))) {
+ const suspensionKind = req._agentEventDetachedActionLifecycle?.readSuspension()?.kind;
+ if (suspensionKind === 'internal_completion') {
+ await GenerationJobManager.updateMetadata(
+ streamId,
+ {
+ agentEventSuspension: {
+ version: actorResult.execution.suspension.version,
+ suspensionId: actorResult.execution.suspension.suspensionId,
+ attempt: actorResult.execution.suspension.attempt,
+ },
+ },
+ job.createdAt,
+ );
+ } else if (!(await client.publishStagedApproval(actorResult.execution.suspension))) {
throw new Error('Re-paused event actor suspension could not be projected to its job');
}
} else if (actorResult.execution.status === 'applied') {
diff --git a/api/server/experimental.js b/api/server/experimental.js
index edd3d84e02..b4083feeda 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -46,6 +46,7 @@ const createValidateImageRequest = require('./middleware/validateImageRequest');
const { startExpiredFileSweep } = require('./services/Files/process');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
const { initializeAgentTriggerService } = require('./services/Agents/triggers');
+const { resumeAgentEventDetachedAction } = require('./services/Agents/detachedActionResume');
const {
recordExpiredScheduleApproval,
initializeScheduleErasureSweep,
@@ -295,7 +296,9 @@ if (cluster.isMaster) {
// its durable run when the generic approval runtime expires it.
GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
GenerationJobManager.setTerminalHostActionHandler(
- createAgentEventTerminalHandler(agentEventMethods),
+ createAgentEventTerminalHandler(agentEventMethods, {
+ resumeDetachedAction: resumeAgentEventDetachedAction,
+ }),
);
GenerationJobManager.initialize();
/**
diff --git a/api/server/index.js b/api/server/index.js
index 8796f3dbca..85ff068b5e 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -67,6 +67,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilities
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
const { initializeAgentTriggerService } = require('./services/Agents/triggers');
+const { resumeAgentEventDetachedAction } = require('./services/Agents/detachedActionResume');
const { initializeScheduleEngine, recordExpiredScheduleApproval } = require('./services/Schedules');
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
const { startExpiredFileSweep } = require('./services/Files/process');
@@ -128,7 +129,9 @@ const configureGenerationStreams = () => {
});
GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);
GenerationJobManager.setTerminalHostActionHandler(
- createAgentEventTerminalHandler(agentEventMethods),
+ createAgentEventTerminalHandler(agentEventMethods, {
+ resumeDetachedAction: resumeAgentEventDetachedAction,
+ }),
);
GenerationJobManager.initialize();
// Stop active generations and close their SSE streams while the HTTP server drains.
diff --git a/api/server/services/Agents/detachedActionResume.js b/api/server/services/Agents/detachedActionResume.js
new file mode 100644
index 0000000000..444584b9f4
--- /dev/null
+++ b/api/server/services/Agents/detachedActionResume.js
@@ -0,0 +1,12 @@
+const { createAgentEventDetachedResumeHandler } = require('@librechat/api');
+const methods = require('~/models');
+
+const resumeAgentEventDetachedAction = createAgentEventDetachedResumeHandler({
+ getAgentTriggerDelivery: methods.getAgentTriggerDelivery,
+ /** Keep server module construction lazy while the typed package owns the
+ * continuation protocol and owner validation. */
+ enqueueAgentTrigger: (envelope, options) =>
+ require('./triggers').enqueueAgentTrigger(envelope, options),
+});
+
+module.exports = { resumeAgentEventDetachedAction };
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index 02e9411929..f6269e2cba 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -3,6 +3,7 @@ const {
createAgentEventContinueResolver,
createSubagentCompletionWakeupResolver,
GenerationJobManager,
+ isAgentEventActorDetachedActionProducerEnabled,
} = require('@librechat/api');
const methods = require('~/models');
@@ -14,6 +15,8 @@ const completionResolver = createSubagentCompletionWakeupResolver({
const service = createAgentTriggerService({
methods,
isPrincipalActive: methods.isAgentTriggerPrincipalActive,
+ supportsDetachedActionCompletion: () =>
+ GenerationJobManager.isRedis && isAgentEventActorDetachedActionProducerEnabled(),
prepareContinue: createAgentEventContinueResolver({
methods,
getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),
diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.js
new file mode 100644
index 0000000000..d824e68182
--- /dev/null
+++ b/api/server/services/Agents/triggers.spec.js
@@ -0,0 +1,54 @@
+const mockCreateAgentTriggerService = jest.fn();
+const mockIsProducerEnabled = jest.fn();
+const mockGenerationJobManager = {
+ isRedis: true,
+ getJob: jest.fn(),
+};
+
+jest.mock('@librechat/api', () => ({
+ createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),
+ createAgentEventContinueResolver: jest.fn(() => jest.fn()),
+ createSubagentCompletionWakeupResolver: jest.fn(() => jest.fn()),
+ GenerationJobManager: mockGenerationJobManager,
+ isAgentEventActorDetachedActionProducerEnabled: () => mockIsProducerEnabled(),
+}));
+
+jest.mock('~/models', () => ({
+ isAgentTriggerPrincipalActive: jest.fn(),
+}));
+
+describe('agent trigger service composition', () => {
+ beforeEach(() => {
+ jest.resetModules();
+ jest.clearAllMocks();
+ mockGenerationJobManager.isRedis = true;
+ mockIsProducerEnabled.mockReturnValue(true);
+ mockCreateAgentTriggerService.mockReturnValue({
+ initialize: jest.fn(),
+ stop: jest.fn(),
+ dispatch: jest.fn(),
+ enqueue: jest.fn(),
+ getDelivery: jest.fn(),
+ getDeliveryStatus: jest.fn(),
+ getDeadLetters: jest.fn(),
+ requeue: jest.fn(),
+ drainUser: jest.fn(),
+ prepareUserPurge: jest.fn(),
+ cancelUserPurge: jest.fn(),
+ purgeUser: jest.fn(),
+ });
+ });
+
+ it('advertises detached completion capability only after Redis and producer activation', () => {
+ require('./triggers');
+ const supportsDetachedActionCompletion =
+ mockCreateAgentTriggerService.mock.calls[0][0].supportsDetachedActionCompletion;
+
+ expect(supportsDetachedActionCompletion()).toBe(true);
+ mockIsProducerEnabled.mockReturnValue(false);
+ expect(supportsDetachedActionCompletion()).toBe(false);
+ mockIsProducerEnabled.mockReturnValue(true);
+ mockGenerationJobManager.isRedis = false;
+ expect(supportsDetachedActionCompletion()).toBe(false);
+ });
+});
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index 41f18cabc3..a60a17dc7a 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -388,6 +388,18 @@ const initializeClient = async ({
});
},
toolEndCallback,
+ /** Bound later by request.js once the authenticated Event Actor owner and
+ * generation fence are known. Ordinary background calls remain unchanged. */
+ eventActorDetachedAction: {
+ reserve: (input) =>
+ req._agentEventDetachedActionLifecycle?.reserve(input) ??
+ Promise.resolve({ status: 'ignored' }),
+ markRunning: (input) =>
+ req._agentEventDetachedActionLifecycle?.markRunning(input) ?? Promise.resolve(false),
+ settle: (input) =>
+ req._agentEventDetachedActionLifecycle?.settle(input) ?? Promise.resolve(false),
+ wake: (input) => req._agentEventDetachedActionLifecycle?.wake(input) ?? Promise.resolve(),
+ },
persistBackgroundCodeResult: createBackgroundCodeResultHandler({
req,
updateToolCallResult: db.updateToolCallResult,
diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts
index 2a32182ee0..22cb2c5519 100644
--- a/packages/api/src/agents/background.spec.ts
+++ b/packages/api/src/agents/background.spec.ts
@@ -1100,6 +1100,80 @@ describe('BackgroundTaskRegistryClass', () => {
expect(atCapacity).toBe(true);
});
+ it('holds capacity permits before task registration and releases them explicitly', () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const permits = Array.from({ length: 10 }, (_, index) =>
+ registry.reserveCapacity({
+ userId: 'u1',
+ conversationId: 'c-permits',
+ toolCallId: `call_${index}`,
+ runId: 'run-1',
+ }),
+ );
+ expect(permits.every((result) => 'permit' in result)).toBe(true);
+ expect(
+ registry.reserveCapacity({
+ userId: 'u1',
+ conversationId: 'c-permits',
+ toolCallId: 'call_rejected',
+ runId: 'run-1',
+ }),
+ ).toEqual({ atCapacity: true });
+ const first = permits[0];
+ if (!('permit' in first)) {
+ throw new Error('expected capacity permit');
+ }
+ registry.releaseCapacity(first.permit);
+ const replacement = registry.reserveCapacity({
+ userId: 'u1',
+ conversationId: 'c-permits',
+ toolCallId: 'call_replacement',
+ runId: 'run-1',
+ });
+ if (!('permit' in replacement)) {
+ throw new Error('expected replacement permit');
+ }
+ const created = registry.create({
+ userId: 'u1',
+ conversationId: 'c-permits',
+ toolCallId: 'call_replacement',
+ toolName: 'search_mcp_docs',
+ runId: 'run-1',
+ capacityPermit: replacement.permit,
+ });
+ expect('atCapacity' in created).toBe(false);
+ });
+
+ it('retains an in-flight capacity permit until durable reservation completes', () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_787_000_000_000);
+ const registry = new BackgroundTaskRegistryClass();
+ const admission = registry.reserveCapacity({
+ userId: 'u1',
+ conversationId: 'c-slow-reservation',
+ toolCallId: 'call_slow',
+ runId: 'run-1',
+ });
+ if (!('permit' in admission)) {
+ throw new Error('expected capacity permit');
+ }
+
+ /** Cross both the former one-minute permit timeout and idle-bucket TTL.
+ * A slow durable reservation still owns this slot until its caller
+ * consumes or releases it. */
+ now.mockReturnValue(1_787_000_000_000 + 7 * 60 * 60 * 1000);
+ const created = registry.create({
+ userId: 'u1',
+ conversationId: 'c-slow-reservation',
+ toolCallId: 'call_slow',
+ toolName: 'search_mcp_docs',
+ runId: 'run-1',
+ capacityPermit: admission.permit,
+ });
+
+ expect('atCapacity' in created).toBe(false);
+ now.mockRestore();
+ });
+
it('evicts oldest settled tasks instead of blocking when the total cap is full', () => {
const registry = new BackgroundTaskRegistryClass();
for (let i = 0; i < 200; i++) {
diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts
index 4b1daa511f..13c15dc687 100644
--- a/packages/api/src/agents/background.ts
+++ b/packages/api/src/agents/background.ts
@@ -634,9 +634,19 @@ interface TaskBucket {
tasks: Map<string, BackgroundTask>;
/** toolCallId -> taskId, for dispatch idempotency across graph re-execution. */
byToolCall: Map<string, string>;
+ /** Caller-owned local permits acquired before a caller persists external
+ * launch authority. They prevent capacity rejection from creating a durable
+ * action that was definitely never launched. */
+ capacityPermits: Map<string, { dedupeKey: string }>;
lastAccess: number;
}
+export interface BackgroundTaskCapacityPermit {
+ id: string;
+ userId: string;
+ conversationId: string;
+}
+
const COMPLETED_TASK_TTL_MS = 60 * 60 * 1000;
const IDLE_BUCKET_TTL_MS = 6 * 60 * 60 * 1000;
/** Max wall-clock a task may stay `running` before being reaped as timed-out,
@@ -742,7 +752,7 @@ export class BackgroundTaskRegistryClass {
}
this.lastGlobalSweepAt = now;
for (const [bucketKey, bucket] of this.buckets) {
- if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS) {
+ if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) {
this.buckets.delete(bucketKey);
continue;
}
@@ -754,13 +764,77 @@ export class BackgroundTaskRegistryClass {
const bucketKey = this.key(userId, conversationId);
let bucket = this.buckets.get(bucketKey);
if (!bucket) {
- bucket = { tasks: new Map(), byToolCall: new Map(), lastAccess: now };
+ bucket = {
+ tasks: new Map(),
+ byToolCall: new Map(),
+ capacityPermits: new Map(),
+ lastAccess: now,
+ };
this.buckets.set(bucketKey, bucket);
}
bucket.lastAccess = now;
return bucket;
}
+ private dedupeKey(params: { toolCallId: string; runId?: string; agentId?: string }): string {
+ return `${params.agentId ?? ''}::${params.runId ?? ''}::${params.toolCallId}`;
+ }
+
+ private runningCount(bucket: TaskBucket): number {
+ let running = 0;
+ for (const task of bucket.tasks.values()) {
+ if (task.status === 'running') {
+ running++;
+ }
+ }
+ return running;
+ }
+
+ /** Acquires process-local capacity before a caller persists launch authority.
+ * The synchronous permit closes the capacity-rejection crash window without
+ * making ordinary background tasks durable. */
+ reserveCapacity(params: {
+ userId: string;
+ conversationId: string;
+ toolCallId: string;
+ runId?: string;
+ agentId?: string;
+ }):
+ | { permit: BackgroundTaskCapacityPermit }
+ | { task: BackgroundTask; isNew: false }
+ | { atCapacity: true } {
+ const now = Date.now();
+ this.sweep(now);
+ const bucket = this.getBucket(params.userId, params.conversationId, now);
+ this.sweepBucketTasks(bucket, now);
+ const dedupeKey = this.dedupeKey(params);
+ const existingId = bucket.byToolCall.get(dedupeKey);
+ const existing = existingId == null ? undefined : bucket.tasks.get(existingId);
+ if (existing != null) {
+ return { task: existing, isNew: false };
+ }
+ if (this.runningCount(bucket) + bucket.capacityPermits.size >= MAX_RUNNING_PER_BUCKET) {
+ return { atCapacity: true };
+ }
+ const permit: BackgroundTaskCapacityPermit = {
+ id: randomUUID(),
+ userId: params.userId,
+ conversationId: params.conversationId,
+ };
+ /** The permit is owned by the in-flight caller until it is consumed or
+ * explicitly released. Expiring it by wall clock could strand a durable
+ * reservation when MongoDB is slow; process death already clears local
+ * permits without pretending the external launch happened. */
+ bucket.capacityPermits.set(permit.id, { dedupeKey });
+ return { permit };
+ }
+
+ releaseCapacity(permit: BackgroundTaskCapacityPermit): void {
+ this.buckets
+ .get(this.key(permit.userId, permit.conversationId))
+ ?.capacityPermits.delete(permit.id);
+ }
+
/**
* Registers a task for a tool call. Returns the existing task (and
* `isNew: false`) only when the SAME run re-dispatched the same `toolCallId`
@@ -774,6 +848,7 @@ export class BackgroundTaskRegistryClass {
* task and hand back a stale/foreign result instead of executing.
*/
create(params: {
+ taskId?: string;
userId: string;
conversationId: string;
toolCallId: string;
@@ -785,29 +860,41 @@ export class BackgroundTaskRegistryClass {
* never settle (reaped as timed out) still take the marker/heal path
* instead of leaving the original card on "running" forever. */
harvestStarted?: boolean;
+ capacityPermit?: BackgroundTaskCapacityPermit;
}): { task: BackgroundTask; isNew: boolean } | { atCapacity: true } {
const now = Date.now();
this.sweep(now);
const bucket = this.getBucket(params.userId, params.conversationId, now);
this.sweepBucketTasks(bucket, now);
- const dedupeKey = `${params.agentId ?? ''}::${params.runId ?? ''}::${params.toolCallId}`;
+ const dedupeKey = this.dedupeKey(params);
const existingId = bucket.byToolCall.get(dedupeKey);
if (existingId) {
const existing = bucket.tasks.get(existingId);
if (existing) {
+ if (params.capacityPermit != null) {
+ this.releaseCapacity(params.capacityPermit);
+ }
return { task: existing, isNew: false };
}
}
- let running = 0;
- for (const task of bucket.tasks.values()) {
- if (task.status === 'running') {
- running++;
+ if (params.capacityPermit != null) {
+ const permit = bucket.capacityPermits.get(params.capacityPermit.id);
+ if (
+ params.capacityPermit.userId !== params.userId ||
+ params.capacityPermit.conversationId !== params.conversationId ||
+ permit?.dedupeKey !== dedupeKey
+ ) {
+ throw new Error('Background task capacity permit is stale');
}
+ bucket.capacityPermits.delete(params.capacityPermit.id);
}
/** Only *running* tasks gate dispatch. */
- if (running >= MAX_RUNNING_PER_BUCKET) {
+ if (
+ params.capacityPermit == null &&
+ this.runningCount(bucket) + bucket.capacityPermits.size >= MAX_RUNNING_PER_BUCKET
+ ) {
return { atCapacity: true };
}
/** The total-tasks cap bounds memory but must NOT block new work: evict the
@@ -829,7 +916,7 @@ export class BackgroundTaskRegistryClass {
}
const task: BackgroundTask = {
- id: randomUUID(),
+ id: params.taskId ?? randomUUID(),
toolName: params.toolName,
toolCallId: params.toolCallId,
messageId: params.messageId,
@@ -1050,7 +1137,9 @@ export class BackgroundTaskRegistryClass {
export const backgroundTaskRegistry = new BackgroundTaskRegistryClass();
/** Content for the synthetic ToolMessage returned when a call is backgrounded. */
-export function buildBackgroundHandleContent(task: BackgroundTask): string {
+export function buildBackgroundHandleContent(
+ task: Pick<BackgroundTask, 'id' | 'toolName' | 'status'>,
+): string {
return JSON.stringify({
background_task_id: task.id,
tool: task.toolName,
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index 7ac1f089d7..5de6e55ebd 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -84,6 +84,350 @@ const runBatch = async (
};
describe('createToolExecuteHandler — background tool calls', () => {
+ it('persists an Event Actor launch before invoking and terminal evidence before wakeup', async () => {
+ const events: string[] = [];
+ let finishTool: ((value: { content: string }) => void) | undefined;
+ const tool = {
+ name: 'submit_move_mcp_chess',
+ description: 'submits a move',
+ schema: z.object({ move: z.string() }),
+ invoke: () => {
+ events.push('invoke');
+ return new Promise<{ content: string }>((resolve) => {
+ finishTool = resolve;
+ });
+ },
+ } as unknown as StructuredToolInterface;
+ let reservations = 0;
+ const eventActorDetachedAction = {
+ reserve: jest.fn(async () => {
+ events.push('reserve');
+ reservations += 1;
+ return {
+ status: reservations === 1 ? ('reserved' as const) : ('replay' as const),
+ taskId: 'event-task-stable-1',
+ idempotencyKey: 'a'.repeat(64),
+ };
+ }),
+ markRunning: jest.fn(async () => {
+ events.push('running');
+ return true;
+ }),
+ settle: jest.fn(async () => {
+ events.push('terminal');
+ return true;
+ }),
+ wake: jest.fn(async () => {
+ events.push('wake');
+ }),
+ };
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ eventActorDetachedAction,
+ });
+ const request = {
+ toolCalls: [
+ {
+ id: 'call-event-detached',
+ name: tool.name,
+ args: { move: 'e4', run_in_background: true },
+ },
+ ],
+ agentId: 'agent-player',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },
+ };
+
+ const first = await runBatch(handler, request);
+
+ expect(JSON.parse(first[0].content)).toMatchObject({
+ status: 'running',
+ background_task_id: 'event-task-stable-1',
+ });
+ expect(events).toEqual(['reserve', 'invoke', 'running']);
+ finishTool?.({ content: 'move accepted' });
+ await flushMicrotasks();
+ await flushMicrotasks();
+ expect(events).toEqual(['reserve', 'invoke', 'running', 'terminal', 'wake']);
+ expect(eventActorDetachedAction.reserve).toHaveBeenCalledWith(
+ expect.objectContaining({ turnId: 'event-response:' }),
+ );
+
+ const replay = await runBatch(handler, request);
+
+ expect(JSON.parse(replay[0].content).background_task_id).toBe('event-task-stable-1');
+ expect(events).toEqual(['reserve', 'invoke', 'running', 'terminal', 'wake', 'reserve']);
+ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(
+ expect.objectContaining({
+ taskId: 'event-task-stable-1',
+ status: 'succeeded',
+ result: 'move accepted',
+ }),
+ );
+ });
+
+ it('records filtered detached output as successful side-effect evidence', async () => {
+ const protectedValue = 'Authorization: Bearer detached-secret';
+ const tool = {
+ name: 'submit_move_mcp_chess',
+ description: 'submits a move',
+ schema: z.object({ move: z.string() }),
+ invoke: jest.fn(async () => ({ content: protectedValue })),
+ } as unknown as StructuredToolInterface;
+ const eventActorDetachedAction = {
+ reserve: jest.fn(async () => ({
+ status: 'reserved' as const,
+ taskId: 'event-task-filtered',
+ idempotencyKey: 'e'.repeat(64),
+ })),
+ markRunning: jest.fn(async () => true),
+ settle: jest.fn(async () => true),
+ wake: jest.fn(async () => undefined),
+ };
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ eventActorDetachedAction,
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-event-filtered',
+ name: tool.name,
+ args: { move: 'e4', run_in_background: true },
+ },
+ ],
+ agentId: 'agent-player',
+ configurable: buildConfig([tool.name], {
+ toolArguments: {
+ pii: { fields: ['output'], starterPatterns: ['bearer_header'] },
+ },
+ }),
+ metadata: { thread_id: 'event-filtered-conversation', run_id: 'event-filtered-turn' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(
+ expect.objectContaining({
+ taskId: 'event-task-filtered',
+ status: 'succeeded',
+ result: expect.stringContaining('content_filter_block'),
+ }),
+ );
+ expect(JSON.stringify(eventActorDetachedAction.settle.mock.calls)).not.toContain(
+ protectedValue,
+ );
+ expect(eventActorDetachedAction.settle).not.toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'failed' }),
+ );
+ expect(eventActorDetachedAction.wake).toHaveBeenCalledTimes(1);
+ });
+
+ it('rejects capacity before creating durable Event Actor launch authority', async () => {
+ let invocations = 0;
+ const tool = {
+ name: 'submit_move_mcp_chess',
+ description: 'submits a move',
+ schema: z.object({ move: z.string() }),
+ invoke: () => {
+ invocations += 1;
+ return new Promise(() => undefined);
+ },
+ } as unknown as StructuredToolInterface;
+ const configurable = {
+ req: {
+ user: { id: 'capacity-user' },
+ body: { conversationId: 'capacity-conversation' },
+ },
+ backgroundToolNames: [tool.name],
+ };
+ const metadata = { thread_id: 'capacity-conversation', run_id: 'capacity-run' };
+ const ordinaryHandler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ });
+ await runBatch(ordinaryHandler, {
+ toolCalls: Array.from({ length: 10 }, (_, index) => ({
+ id: `capacity-${index}`,
+ name: tool.name,
+ args: { move: 'e4', run_in_background: true },
+ })),
+ agentId: 'agent-player',
+ configurable,
+ metadata,
+ });
+
+ const reserve = jest.fn(async () => ({
+ status: 'reserved' as const,
+ taskId: 'event-task-capacity',
+ idempotencyKey: 'f'.repeat(64),
+ }));
+ const detachedHandler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ eventActorDetachedAction: {
+ reserve,
+ markRunning: jest.fn(async () => true),
+ settle: jest.fn(async () => true),
+ wake: jest.fn(async () => undefined),
+ },
+ });
+
+ const [rejected] = await runBatch(detachedHandler, {
+ toolCalls: [
+ {
+ id: 'capacity-detached',
+ name: tool.name,
+ args: { move: 'e5', run_in_background: true },
+ },
+ ],
+ agentId: 'agent-player',
+ configurable,
+ metadata,
+ });
+
+ expect(rejected.content).toContain('Too many background tasks');
+ expect(invocations).toBe(10);
+ expect(reserve).not.toHaveBeenCalled();
+ });
+
+ it('does not launch an expected action when staged production is disabled', async () => {
+ const invoke = jest.fn(async () => ({ content: 'move accepted' }));
+ const tool = {
+ name: 'submit_move_mcp_chess',
+ description: 'submits a move',
+ schema: z.object({ move: z.string() }),
+ invoke,
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ eventActorDetachedAction: {
+ reserve: jest.fn(async () => ({
+ status: 'conflict' as const,
+ error: 'Detached Event Actor production is not activated',
+ })),
+ markRunning: jest.fn(async () => true),
+ settle: jest.fn(async () => true),
+ wake: jest.fn(async () => undefined),
+ },
+ });
+
+ await expect(
+ runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-disabled',
+ name: tool.name,
+ args: { move: 'e4', run_in_background: true },
+ },
+ ],
+ agentId: 'agent-player',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },
+ }),
+ ).resolves.toEqual([
+ expect.objectContaining({ content: '', errorMessage: expect.stringContaining('activated') }),
+ ]);
+ expect(invoke).not.toHaveBeenCalled();
+ });
+
+ it('persists a synchronous detached-tool rejection as terminal evidence', async () => {
+ const tool = {
+ name: 'submit_move_mcp_chess',
+ description: 'submits a move',
+ schema: z.object({ move: z.string() }),
+ invoke: () => {
+ throw new Error('launch rejected synchronously');
+ },
+ } as unknown as StructuredToolInterface;
+ const eventActorDetachedAction = {
+ reserve: jest.fn(async () => ({
+ status: 'reserved' as const,
+ taskId: 'event-task-sync-error',
+ idempotencyKey: 'b'.repeat(64),
+ })),
+ markRunning: jest.fn(async () => true),
+ settle: jest.fn(async () => true),
+ wake: jest.fn(async () => undefined),
+ };
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ eventActorDetachedAction,
+ });
+
+ await expect(
+ runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-sync-error',
+ name: tool.name,
+ args: { move: 'e4', run_in_background: true },
+ },
+ ],
+ agentId: 'agent-player',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },
+ }),
+ ).resolves.toEqual([
+ expect.objectContaining({ content: expect.stringContaining('event-task-sync-error') }),
+ ]);
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(
+ expect.objectContaining({
+ taskId: 'event-task-sync-error',
+ status: 'failed',
+ error: expect.stringContaining('launch rejected synchronously'),
+ }),
+ );
+ expect(eventActorDetachedAction.wake).toHaveBeenCalledTimes(1);
+ });
+
+ it('records an aborted detached expected action as cancelled', async () => {
+ const abortError = Object.assign(new Error('operation aborted'), { name: 'AbortError' });
+ const tool = {
+ name: 'submit_move_mcp_chess',
+ description: 'submits a move',
+ schema: z.object({ move: z.string() }),
+ invoke: async () => {
+ throw abortError;
+ },
+ } as unknown as StructuredToolInterface;
+ const eventActorDetachedAction = {
+ reserve: jest.fn(async () => ({
+ status: 'reserved' as const,
+ taskId: 'event-task-cancelled',
+ idempotencyKey: 'c'.repeat(64),
+ })),
+ markRunning: jest.fn(async () => true),
+ settle: jest.fn(async () => true),
+ wake: jest.fn(async () => undefined),
+ };
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ eventActorDetachedAction,
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-cancelled',
+ name: tool.name,
+ args: { move: 'e4', run_in_background: true },
+ },
+ ],
+ agentId: 'agent-player',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'cancelled', error: 'operation aborted' }),
+ );
+ });
+
it('returns a handle immediately, runs the tool once detached, and yields the result via check_background_task', async () => {
const state = { calls: 0 } as { calls: number; lastInput?: Record<string, unknown> };
const searchTool = makeSearchTool(state);
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index a87f7b2bb1..f11d891df3 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -105,6 +105,40 @@ export interface ToolEndCallbackData {
};
}
+export interface EventActorDetachedActionLifecycle {
+ reserve(input: {
+ toolName: string;
+ toolCallId: string;
+ turnId: string;
+ arguments: unknown;
+ }): Promise<
+ | { status: 'ignored' }
+ | { status: 'conflict'; error?: string }
+ | {
+ status: 'terminal';
+ taskId: string;
+ idempotencyKey: string;
+ outcome: 'succeeded' | 'failed' | 'cancelled';
+ result?: string;
+ error?: string;
+ }
+ | {
+ status: 'reserved' | 'replay';
+ taskId: string;
+ idempotencyKey: string;
+ }
+ >;
+ markRunning(input: { taskId: string; idempotencyKey: string }): Promise<boolean>;
+ settle(input: {
+ taskId: string;
+ idempotencyKey: string;
+ status: 'succeeded' | 'failed' | 'cancelled';
+ result?: unknown;
+ error?: string;
+ }): Promise<boolean>;
+ wake(input: { taskId: string; idempotencyKey: string }): Promise<void>;
+}
+
export interface ToolEndCallbackMetadata {
run_id?: string;
thread_id?: string;
@@ -134,6 +168,8 @@ export interface ToolExecuteOptions {
subagentTasks?: SubagentTaskConfig;
/** Callback to process tool artifacts (code output files, file citations, etc.) */
toolEndCallback?: ToolEndCallback;
+ /** Durable internal-completion adapter, present only for an Event Actor invocation. */
+ eventActorDetachedAction?: EventActorDetachedActionLifecycle;
/**
* Persists a backgrounded code-execution result onto the dispatch turn once
* the detached call settles: downloads/persists generated files, patches the
@@ -4232,6 +4268,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const {
loadTools,
toolEndCallback,
+ eventActorDetachedAction,
persistBackgroundCodeResult,
emitAttachment,
emitPtcProgress,
@@ -4367,7 +4404,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const backgroundRunId = (metadata as Record<string, unknown>)?.run_id as
| string
| undefined;
- const dispatchBackgroundToolCall = (tc: ToolCallRequest): ToolExecuteResult => {
+ const dispatchBackgroundToolCall = async (
+ tc: ToolCallRequest,
+ ): Promise<ToolExecuteResult> => {
/** A tool that failed to load must error immediately (matching the
* foreground path) — a synthetic "started" handle would tell the
* model a side effect is in flight that never executed. */
@@ -4393,7 +4432,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
if (filtered != null) {
return filtered;
}
- const created = backgroundTaskRegistry.create({
+ const registration = {
userId: backgroundUserId,
conversationId: backgroundConversationId,
toolCallId: tc.id,
@@ -4405,8 +4444,93 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
* starts a fresh task instead of colliding. */
agentId,
runId: `${backgroundRunId ?? ''}:${tc.turn ?? ''}`,
+ };
+ const capacityAdmission =
+ eventActorDetachedAction == null
+ ? undefined
+ : backgroundTaskRegistry.reserveCapacity(registration);
+ if (capacityAdmission != null && 'atCapacity' in capacityAdmission) {
+ return {
+ toolCallId: tc.id,
+ status: 'success' as const,
+ content: buildBackgroundCapacityContent(tc.name),
+ };
+ }
+ const capacityPermit =
+ capacityAdmission != null && 'permit' in capacityAdmission
+ ? capacityAdmission.permit
+ : undefined;
+ let detachedReservation;
+ try {
+ detachedReservation = await eventActorDetachedAction?.reserve({
+ toolName: tc.name,
+ toolCallId: tc.id,
+ turnId: registration.runId,
+ arguments: normalizedArgs,
+ });
+ } catch (error) {
+ if (capacityPermit != null) {
+ backgroundTaskRegistry.releaseCapacity(capacityPermit);
+ }
+ throw error;
+ }
+ if (detachedReservation?.status === 'conflict') {
+ if (capacityPermit != null) {
+ backgroundTaskRegistry.releaseCapacity(capacityPermit);
+ }
+ return {
+ toolCallId: tc.id,
+ status: 'error' as const,
+ content: '',
+ errorMessage:
+ detachedReservation.error ??
+ 'Detached Event Actor action conflicts with its durable launch authority',
+ };
+ }
+ if (detachedReservation?.status === 'terminal') {
+ if (capacityPermit != null) {
+ backgroundTaskRegistry.releaseCapacity(capacityPermit);
+ }
+ if (detachedReservation.outcome === 'succeeded') {
+ return {
+ toolCallId: tc.id,
+ status: 'success' as const,
+ content: detachedReservation.result ?? '',
+ };
+ }
+ return {
+ toolCallId: tc.id,
+ status: 'error' as const,
+ content: '',
+ errorMessage:
+ detachedReservation.error ?? `Detached action ${detachedReservation.outcome}`,
+ };
+ }
+ if (detachedReservation?.status === 'replay') {
+ if (capacityPermit != null) {
+ backgroundTaskRegistry.releaseCapacity(capacityPermit);
+ }
+ return {
+ toolCallId: tc.id,
+ status: 'success' as const,
+ content: buildBackgroundHandleContent({
+ id: detachedReservation.taskId,
+ toolName: tc.name,
+ status: 'running',
+ }),
+ };
+ }
+ const created = backgroundTaskRegistry.create({
+ ...(detachedReservation?.status === 'reserved'
+ ? { taskId: detachedReservation.taskId }
+ : {}),
+ ...registration,
+ ...(capacityPermit == null ? {} : { capacityPermit }),
});
if ('atCapacity' in created) {
+ if (detachedReservation?.status === 'reserved') {
+ throw new Error('Detached Event Actor lost its pre-admitted background capacity');
+ }
return {
toolCallId: tc.id,
status: 'success' as const,
@@ -4495,17 +4619,74 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
})();
};
- void (async () => {
- try {
- const result = (await tool.invoke(normalizedArgs, {
+ let invokePromise: Promise<{ content?: unknown; artifact?: unknown }>;
+ try {
+ invokePromise = Promise.resolve(
+ tool.invoke(normalizedArgs, {
/** Full invoke config (not just identity): a detached
* code call still needs `session_id`/`_injected_files`/
* `_runtime_session_hint` or it runs fileless on the
* Code API's default runtime session. */
toolCall: buildToolCallConfig(tc, mergedConfigurable),
- configurable: mergedConfigurable,
+ configurable: {
+ ...mergedConfigurable,
+ ...(detachedReservation?.status === 'reserved'
+ ? {
+ eventActorDetachedAction: {
+ taskId: detachedReservation.taskId,
+ idempotencyKey: detachedReservation.idempotencyKey,
+ },
+ }
+ : {}),
+ },
metadata,
- } as Record<string, unknown>)) as { content?: unknown; artifact?: unknown };
+ } as Record<string, unknown>),
+ ) as Promise<{ content?: unknown; artifact?: unknown }>;
+ } catch (error) {
+ /** Structured tools are permitted to reject synchronously.
+ * Preserve the durable reservation and route that rejection
+ * through the same terminal-evidence path as an async one. */
+ invokePromise = Promise.reject(error);
+ }
+ const persistDetachedTerminal = async (
+ input:
+ | { status: 'succeeded'; result: unknown }
+ | { status: 'failed' | 'cancelled'; error: string },
+ ): Promise<boolean> => {
+ if (
+ detachedReservation?.status !== 'reserved' ||
+ eventActorDetachedAction == null
+ ) {
+ return true;
+ }
+ return eventActorDetachedAction.settle({
+ taskId: detachedReservation.taskId,
+ idempotencyKey: detachedReservation.idempotencyKey,
+ ...input,
+ });
+ };
+ const wakeDetachedActor = async (): Promise<void> => {
+ if (
+ detachedReservation?.status !== 'reserved' ||
+ eventActorDetachedAction == null
+ ) {
+ return;
+ }
+ try {
+ await eventActorDetachedAction.wake({
+ taskId: detachedReservation.taskId,
+ idempotencyKey: detachedReservation.idempotencyKey,
+ });
+ } catch (wakeError) {
+ logger.warn(
+ `[event-actor] Failed to wake detached action ${detachedReservation.taskId}`,
+ wakeError,
+ );
+ }
+ };
+ void (async () => {
+ try {
+ const result = await invokePromise;
if (isCodeCall) {
markCodeSandboxWarm();
}
@@ -4523,6 +4704,14 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const errorOutput = isCodeCall
? toCodeToolFailure(tc.name, policyError)
: policyError;
+ if (
+ !(await persistDetachedTerminal({
+ status: 'succeeded',
+ result: errorOutput,
+ }))
+ ) {
+ return;
+ }
backgroundTaskRegistry.fail(
backgroundUserId,
backgroundConversationId,
@@ -4531,6 +4720,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
{ harvestStarted: harvestEnabled },
);
harvestCodeResult({ output: errorOutput });
+ await wakeDetachedActor();
return;
}
/** Hold any artifact (images, files, UI resources,
@@ -4540,6 +4730,14 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
* artifactPromises are already awaited and the stream is
* closed, so that push would be silently dropped. The poll
* turn delivers it live in `check_background_task`. */
+ if (
+ !(await persistDetachedTerminal({
+ status: 'succeeded',
+ result: content,
+ }))
+ ) {
+ return;
+ }
backgroundTaskRegistry.complete(
backgroundUserId,
backgroundConversationId,
@@ -4550,6 +4748,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
output: typeof content === 'string' ? content : undefined,
artifact: result.artifact,
});
+ await wakeDetachedActor();
} catch (toolError) {
const policyError =
toolError instanceof ContentFilterError
@@ -4569,6 +4768,20 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
isCodeCall && (policyError != null || filteredError != null)
? toCodeToolFailure(tc.name, neutralizedError)
: neutralizedError;
+ const detachedTerminalStatus =
+ toolError instanceof Error &&
+ (toolError.name === 'AbortError' ||
+ (toolError as Error & { code?: string }).code === 'ABORT_ERR')
+ ? 'cancelled'
+ : 'failed';
+ if (
+ !(await persistDetachedTerminal({
+ status: detachedTerminalStatus,
+ error: deliveredError,
+ }))
+ ) {
+ return;
+ }
backgroundTaskRegistry.fail(
backgroundUserId,
backgroundConversationId,
@@ -4580,8 +4793,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
{ harvestStarted: harvestEnabled },
);
harvestCodeResult({ output: deliveredError });
+ await wakeDetachedActor();
}
})();
+ if (
+ detachedReservation?.status === 'reserved' &&
+ eventActorDetachedAction != null &&
+ !(await eventActorDetachedAction.markRunning({
+ taskId: detachedReservation.taskId,
+ idempotencyKey: detachedReservation.idempotencyKey,
+ }))
+ ) {
+ throw new Error('Detached Event Actor launch acknowledgement is stale');
+ }
}
return {
toolCallId: tc.id,
@@ -4836,7 +5060,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
persistBackgroundCodeResult == null
)
) {
- return reportResult(dispatchBackgroundToolCall(tc));
+ return reportResult(await dispatchBackgroundToolCall(tc));
}
const execute = async (
diff --git a/packages/api/src/agents/plan.spec.ts b/packages/api/src/agents/plan.spec.ts
index e98317ff26..dff85c32aa 100644
--- a/packages/api/src/agents/plan.spec.ts
+++ b/packages/api/src/agents/plan.spec.ts
@@ -7,7 +7,6 @@ const baseInput = (): ResolveAgentTurnExecutionPlanInput => ({
isNewConversation: false,
canPause: false,
durableEventActorSuspensions: false,
- expectedActionMayDetach: false,
});
const boundEvent = () => ({
@@ -67,7 +66,6 @@ describe('resolveAgentTurnExecutionPlan', () => {
['no binding', { event: { ...boundEvent(), binding: undefined } }],
['no expected action', { event: { ...boundEvent(), expectedAction: undefined } }],
['memory checkpointer', { event: boundEvent(), checkpointerType: 'memory' }],
- ['detachable action', { event: boundEvent(), expectedActionMayDetach: true }],
[
'pre-cutover pause consumer fleet',
{ event: boundEvent(), canPause: true, durableEventActorSuspensions: false },
diff --git a/packages/api/src/agents/plan.ts b/packages/api/src/agents/plan.ts
index c15c88d69f..ff47d60c4d 100644
--- a/packages/api/src/agents/plan.ts
+++ b/packages/api/src/agents/plan.ts
@@ -38,7 +38,6 @@ export interface ResolveAgentTurnExecutionPlanInput {
* after every resume consumer understands signed Event Actor suspensions. */
durableEventActorSuspensions: boolean;
checkpointerType?: TCheckpointerType;
- expectedActionMayDetach: boolean;
}
const COMPLETION_EVENT_TYPE = 'subagent.completion';
@@ -73,7 +72,6 @@ export function resolveAgentTurnExecutionPlan(
binding != null &&
expectedAction != null &&
input.checkpointerType !== 'memory' &&
- !input.expectedActionMayDetach &&
(!input.canPause || input.durableEventActorSuspensions);
let strategy: AgentTurnContinuationStrategy = 'history';
if (input.isNewConversation) {
diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md
index 55e60a9f1b..b18f4108ce 100644
--- a/packages/api/src/agents/triggers/README.md
+++ b/packages/api/src/agents/triggers/README.md
@@ -127,6 +127,17 @@ Bound child continuations are automatic after authentication and binding authori
`endpoints.agents.eventDriven.selfUrl`; most deployments should omit both and use the bound
listener.
+Detached Event Actor completion production has an explicit rolling-deployment fence. It remains
+fail-closed until Redis generation streams are active and
+`AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED=true` is set. Deploy the new version to every
+replica first, verify that no old API or worker replica remains, and only then enable the producer
+across the Redis-backed fleet. This ordering ensures Redis recovery, HTTP completion consumers,
+account-deletion drains, lane maintenance, and operational requeue/dead-letter paths all understand
+capability-fenced completion work before any is created. For rollback, disable the producer, drain
+all detached completion work, and only then start an older version. Internal detached completions
+always target the current replica's bound listener; `AGENT_TRIGGERS_SELF_URL` remains available for
+ordinary trigger dispatch but cannot route capability-owned completion work to another replica.
+
```http
POST /api/agents/v1/events/bindings
Authorization: Bearer <remote-agents-api-key>
diff --git a/packages/api/src/agents/triggers/actor.spec.ts b/packages/api/src/agents/triggers/actor.spec.ts
index 4a429ff61d..64cc5d6998 100644
--- a/packages/api/src/agents/triggers/actor.spec.ts
+++ b/packages/api/src/agents/triggers/actor.spec.ts
@@ -495,6 +495,114 @@ describe('event actor host adapter', () => {
expect(dependencies.commitState).not.toHaveBeenCalled();
});
+ it('carries applied expected-action evidence across a later re-pause', async () => {
+ let storedSuspension: IAgentEventActorSuspension | undefined;
+ let pendingPause:
+ | {
+ actionId: string;
+ jobCreatedAt: number;
+ interrupt: EventActorInterrupt;
+ }
+ | undefined;
+ let observedAction: { toolName: string; toolCallId?: string } | undefined;
+ const dependencies = {
+ ...deps(),
+ storeSuspension: jest.fn(async (input) => {
+ storedSuspension = {
+ suspension: input.suspension,
+ actionId: input.actionId,
+ jobCreatedAt: input.jobCreatedAt,
+ appliedAction: input.appliedAction,
+ status: 'pending',
+ observedAt: new Date(),
+ };
+ return { status: 'stored' as const };
+ }),
+ claimSuspension: jest.fn(async ({ resumeAttemptId }) => {
+ if (storedSuspension == null) {
+ throw new Error('test suspension was not stored');
+ }
+ storedSuspension = { ...storedSuspension, status: 'claimed', resumeAttemptId };
+ return { status: 'claimed' as const };
+ }),
+ settleSuspension: jest.fn(async () => ({ status: 'settled' as const })),
+ };
+ dependencies.getSnapshot.mockImplementation(async () => ({
+ state,
+ reconciliations: [],
+ legacyTurn: null,
+ suspension: storedSuspension ?? null,
+ epoch,
+ }));
+ const initial = await executeAgentEventActor(
+ {
+ user: 'user-1',
+ conversationId,
+ invocationId: 'event-action-repause',
+ event: { id: 'event-action-repause' },
+ signal: new AbortController().signal,
+ invoke: async () => 'initial-pause',
+ readAppliedAction: () => undefined,
+ readSuspension: () => ({
+ actionId: 'detached-task',
+ jobCreatedAt: 801,
+ interrupt: { id: 'detached-task', payload: { type: 'detached' } },
+ }),
+ },
+ dependencies,
+ );
+ if (initial.execution.status !== 'suspended') {
+ throw new Error('test setup did not suspend');
+ }
+ observedAction = { toolName: 'submit_move', toolCallId: 'call-detached' };
+ pendingPause = {
+ actionId: 'ask-user',
+ jobCreatedAt: 802,
+ interrupt: { id: 'ask-user', payload: { type: 'ask_user_question' } },
+ };
+ const repaused = await resumeAgentEventActor(
+ {
+ user: 'user-1',
+ conversationId,
+ suspension: initial.execution.suspension,
+ resumeAttemptId: 'resume-detached',
+ resumeValue: { status: 'succeeded' },
+ signal: new AbortController().signal,
+ resume: async () => 'asks-user',
+ readAppliedAction: () => observedAction,
+ readSuspension: () => pendingPause,
+ },
+ dependencies,
+ );
+ expect(repaused.execution.status).toBe('suspended');
+ expect(dependencies.storeSuspension).toHaveBeenLastCalledWith(
+ expect.objectContaining({ appliedAction: observedAction }),
+ );
+ if (repaused.execution.status !== 'suspended') {
+ throw new Error('test setup did not re-pause');
+ }
+ observedAction = undefined;
+ pendingPause = undefined;
+ const completed = await resumeAgentEventActor(
+ {
+ user: 'user-1',
+ conversationId,
+ suspension: repaused.execution.suspension,
+ resumeAttemptId: 'resume-human',
+ resumeValue: { answer: 'continue' },
+ signal: new AbortController().signal,
+ resume: async () => 'completed',
+ readAppliedAction: () => observedAction,
+ readSuspension: () => pendingPause,
+ },
+ dependencies,
+ );
+ expect(completed.execution).toMatchObject({
+ status: 'applied',
+ result: { action: { toolName: 'submit_move', toolCallId: 'call-detached' } },
+ });
+ });
+
it('cold-starts once, then forks and warm-continues only the next event', async () => {
const dependencies = deps();
const invocations: Array<{ continuation: string; checkpointId?: string }> = [];
diff --git a/packages/api/src/agents/triggers/actor.ts b/packages/api/src/agents/triggers/actor.ts
index 985fc0ca8c..ec5bdb39ac 100644
--- a/packages/api/src/agents/triggers/actor.ts
+++ b/packages/api/src/agents/triggers/actor.ts
@@ -64,6 +64,7 @@ export interface ExecuteAgentEventActorInput<T> {
readAppliedAction(): AgentEventAppliedAction | undefined;
readSuspension?():
| {
+ kind?: 'human_decision' | 'internal_completion';
actionId: string;
jobCreatedAt: number;
interrupt: EventActorInterrupt;
@@ -118,6 +119,7 @@ export interface ResumeAgentEventActorInput<T> {
readAppliedAction(): AgentEventAppliedAction | undefined;
readSuspension?():
| {
+ kind?: 'human_decision' | 'internal_completion';
actionId: string;
jobCreatedAt: number;
interrupt: EventActorInterrupt;
@@ -237,7 +239,14 @@ export async function executeAgentEventActor<T>(
let preparedContext: AgentEventActorContext | undefined;
let resultContext: AgentEventActorContext | undefined;
let pendingSuspension:
- | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt }
+ | {
+ kind?: 'human_decision' | 'internal_completion';
+ appliedAction?: AgentEventAppliedAction;
+ handlingGenerationCreatedAt?: number;
+ actionId: string;
+ jobCreatedAt: number;
+ interrupt: EventActorInterrupt;
+ }
| undefined;
let actionAppliedBeforePause = false;
const adapter: EventActorHostAdapter<EventActorEvent, EventActorResult> = {
@@ -617,6 +626,10 @@ export async function executeAgentEventActor<T>(
conversationId: input.conversationId,
...(input.tenantId == null ? {} : { tenantId: input.tenantId }),
suspension: request.suspension as IAgentEventActorSuspensionEvidence,
+ ...(pendingSuspension.kind == null ? {} : { kind: pendingSuspension.kind }),
+ ...(pendingSuspension.handlingGenerationCreatedAt == null
+ ? {}
+ : { handlingGenerationCreatedAt: pendingSuspension.handlingGenerationCreatedAt }),
actionId: pendingSuspension.actionId,
jobCreatedAt: pendingSuspension.jobCreatedAt,
...(actionAppliedBeforePause ? { invalidateHead: true } : {}),
@@ -900,7 +913,14 @@ export async function resumeAgentEventActor<T>(
let observedEpoch: number | undefined;
let resultContext: AgentEventActorContext | undefined;
let pendingSuspension:
- | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt }
+ | {
+ kind?: 'human_decision' | 'internal_completion';
+ appliedAction?: AgentEventAppliedAction;
+ handlingGenerationCreatedAt?: number;
+ actionId: string;
+ jobCreatedAt: number;
+ interrupt: EventActorInterrupt;
+ }
| undefined;
let actionAppliedBeforePause = false;
@@ -979,9 +999,17 @@ export async function resumeAgentEventActor<T>(
* human boundary. Publish the successor suspension before considering
* the segment terminal; otherwise the successor checkpoint is committed
* without any resumable host action. */
- pendingSuspension = input.readSuspension?.();
- if (pendingSuspension != null) {
- actionAppliedBeforePause = input.readAppliedAction() != null;
+ const observedAction = input.readAppliedAction() ?? hostSuspension.appliedAction;
+ const action = observedAction == null ? undefined : toEventActorAppliedAction(observedAction);
+ const observedSuspension = input.readSuspension?.();
+ if (observedSuspension != null) {
+ pendingSuspension = {
+ ...observedSuspension,
+ ...(observedAction == null ? {} : { appliedAction: observedAction }),
+ handlingGenerationCreatedAt:
+ hostSuspension.handlingGenerationCreatedAt ?? hostSuspension.jobCreatedAt,
+ };
+ actionAppliedBeforePause = observedAction != null;
const checkpoint = await captureAgentEventCheckpoint(
input.conversationId,
request.suspension.checkpoint.checkpointNs,
@@ -1006,8 +1034,6 @@ export async function resumeAgentEventActor<T>(
},
};
}
- const observedAction = input.readAppliedAction();
- const action = observedAction == null ? undefined : toEventActorAppliedAction(observedAction);
if (action == null) {
if (invocationError != null) {
return { status: 'claimed_failed', error: asError(invocationError) };
@@ -1079,6 +1105,13 @@ export async function resumeAgentEventActor<T>(
return deps.storeSuspension({
...owner,
suspension: request.suspension as IAgentEventActorSuspensionEvidence,
+ ...(pendingSuspension.kind == null ? {} : { kind: pendingSuspension.kind }),
+ ...(pendingSuspension.appliedAction == null
+ ? {}
+ : { appliedAction: pendingSuspension.appliedAction }),
+ ...(pendingSuspension.handlingGenerationCreatedAt == null
+ ? {}
+ : { handlingGenerationCreatedAt: pendingSuspension.handlingGenerationCreatedAt }),
actionId: pendingSuspension.actionId,
jobCreatedAt: pendingSuspension.jobCreatedAt,
...(actionAppliedBeforePause ? { invalidateHead: true } : {}),
diff --git a/packages/api/src/agents/triggers/bindingResolver.spec.ts b/packages/api/src/agents/triggers/bindingResolver.spec.ts
index 9e21bd1bfe..5ec80fb47c 100644
--- a/packages/api/src/agents/triggers/bindingResolver.spec.ts
+++ b/packages/api/src/agents/triggers/bindingResolver.spec.ts
@@ -1,5 +1,9 @@
import { AIMessage } from '@langchain/core/messages';
import type { IConversation } from '@librechat/data-schemas';
+import {
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+} from './detachedAction';
import { createAgentTriggerEnvelope, type AgentContinueTriggerEnvelope } from './envelope';
import { createAgentEventContinueResolver } from './bindingResolver';
@@ -30,6 +34,62 @@ function envelope(): AgentContinueTriggerEnvelope {
}) as AgentContinueTriggerEnvelope;
}
+function detachedCompletionEnvelope(generationCreatedAt: number): AgentContinueTriggerEnvelope {
+ return createAgentTriggerEnvelope({
+ mode: 'continue',
+ requestId: 'request-completion',
+ deliveryId: 'detached_completion:task-1',
+ receivedAt: 2,
+ principal: { id: 'user-1', tenantId: 'tenant-1' },
+ event: {
+ id: 'task-1',
+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ occurredAt: 2,
+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },
+ payload: {
+ version: 1,
+ invocationId: 'delivery-1',
+ generationCreatedAt,
+ wakeGenerationCreatedAt: generationCreatedAt,
+ taskId: 'task-1',
+ idempotencyKey: 'a'.repeat(64),
+ },
+ },
+ input: 'Detached completion.',
+ target: {
+ agentId: 'agent-player',
+ conversationId: 'child-thread',
+ parentMessageId: 'placeholder',
+ bindingId,
+ sourceKeyId,
+ },
+ }) as AgentContinueTriggerEnvelope;
+}
+
+function boundMethods() {
+ return {
+ getAgentEventBinding: jest.fn(async () => ({
+ conversationId: 'child-thread',
+ agentId: 'agent-player',
+ tenantId: 'tenant-1',
+ binding: { bindingId, sourceKeyId, actorId: 'player' },
+ lineage: {
+ parentConversationId: 'parent-thread',
+ parentAgentId: 'agent-director',
+ } as never,
+ })),
+ getConvo: jest.fn(
+ async () =>
+ ({
+ conversationId: 'parent-thread',
+ agent_id: 'agent-director',
+ tenantId: 'tenant-1',
+ }) as IConversation,
+ ),
+ getMessages: jest.fn(async () => []) as never,
+ };
+}
+
describe('agent event continuation resolver', () => {
it('re-resolves the latest assistant leaf immediately before dispatch', async () => {
const getMessages = jest.fn(async () => [
@@ -129,6 +189,33 @@ describe('agent event continuation resolver', () => {
});
});
+ it('admits only the exact detached completion through its terminal host-action fence', async () => {
+ const createdAt = 77;
+ const resolver = createAgentEventContinueResolver({
+ getGenerationJob: jest.fn(async () => ({
+ status: 'complete',
+ createdAt,
+ metadata: { terminalPersistencePending: true },
+ })),
+ methods: boundMethods(),
+ });
+
+ await expect(
+ resolver(detachedCompletionEnvelope(createdAt), {
+ idempotencyKey: 'completion-1',
+ } as never),
+ ).resolves.toMatchObject({ status: 'ready' });
+
+ await expect(
+ resolver(detachedCompletionEnvelope(createdAt + 1), {
+ idempotencyKey: 'completion-stale',
+ } as never),
+ ).rejects.toMatchObject({
+ code: 'EVENT_ACTOR_NOT_READY',
+ deferWithoutAttempt: true,
+ });
+ });
+
it('fails closed after the binding parent is removed', async () => {
const resolver = createAgentEventContinueResolver({
methods: {
diff --git a/packages/api/src/agents/triggers/bindingResolver.ts b/packages/api/src/agents/triggers/bindingResolver.ts
index 02c9e26483..cfa964abf0 100644
--- a/packages/api/src/agents/triggers/bindingResolver.ts
+++ b/packages/api/src/agents/triggers/bindingResolver.ts
@@ -3,6 +3,11 @@ import type { ConversationMethods, MessageMethods } from '@librechat/data-schema
import type { AgentTriggerContinuePreparation, AgentTriggerExecutionHostDeps } from './host';
import type { AgentContinueTriggerEnvelope } from './envelope';
import type { AgentTriggerDispatchContext } from './dispatch';
+import {
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ parseAgentEventActorDetachedCompletion,
+} from './detachedAction';
import { isAgentEventRetentionActive } from '../eventRetention';
import { AgentTriggerExecutionError } from './host';
@@ -14,6 +19,7 @@ export interface AgentEventContinueResolverDeps {
getGenerationJob?: (conversationId: string) => Promise<
| {
status?: string;
+ createdAt?: number;
metadata?: { terminalPersistencePending?: boolean };
}
| null
@@ -108,10 +114,19 @@ export function createAgentEventContinueResolver({
true,
);
}
+ const detachedCompletion =
+ envelope.event.type === EVENT_ACTOR_DETACHED_COMPLETION_TYPE &&
+ envelope.event.source.type === 'internal' &&
+ envelope.event.source.id === EVENT_ACTOR_DETACHED_COMPLETION_SOURCE
+ ? parseAgentEventActorDetachedCompletion(envelope.event.payload)
+ : undefined;
+ const ownsTerminalWake =
+ detachedCompletion != null &&
+ active?.createdAt === detachedCompletion.wakeGenerationCreatedAt;
if (
active?.status === 'running' ||
active?.status === 'requires_action' ||
- active?.metadata?.terminalPersistencePending === true
+ (active?.metadata?.terminalPersistencePending === true && !ownsTerminalWake)
) {
throw new AgentTriggerExecutionError('The event actor is still handling an earlier turn.', {
mode: 'continue',
diff --git a/packages/api/src/agents/triggers/delivery.ts b/packages/api/src/agents/triggers/delivery.ts
index 33c0e2d345..4920033a47 100644
--- a/packages/api/src/agents/triggers/delivery.ts
+++ b/packages/api/src/agents/triggers/delivery.ts
@@ -21,6 +21,9 @@ export interface AgentTriggerEnqueueOptions {
/** Opt-in only for observational bound-child continuations. Actionable,
* fenced, approval, HITL, and control deliveries must remain individual. */
coalesce?: AgentTriggerCoalesceOptions;
+ /** Server-owned capability fence. Deliveries carrying this marker remain
+ * invisible to older workers during a rolling deployment. */
+ requiredWorkerCapability?: string;
}
export interface PreparedAgentTriggerDelivery {
@@ -38,6 +41,7 @@ export interface PreparedAgentTriggerDelivery {
/** Persisted rollout marker: keep this bound actor lane queued until the
* admitted child turn records an authoritative terminal outcome. */
awaitTerminalHandling?: boolean;
+ requiredWorkerCapability?: string;
}
export class AgentTriggerDeliveryError extends TypeError {
@@ -188,6 +192,9 @@ export function prepareAgentTriggerDelivery(
...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
availableAt: coalesceUntil ?? requestedAvailableAt,
envelopeBytes: bytes,
+ ...(options.requiredWorkerCapability == null
+ ? {}
+ : { requiredWorkerCapability: options.requiredWorkerCapability }),
...(coalesceKey != null &&
coalesceUntil != null && {
coalesceKey,
diff --git a/packages/api/src/agents/triggers/detachedAction.spec.ts b/packages/api/src/agents/triggers/detachedAction.spec.ts
new file mode 100644
index 0000000000..c5df8e949a
--- /dev/null
+++ b/packages/api/src/agents/triggers/detachedAction.spec.ts
@@ -0,0 +1,551 @@
+import type { AgentTriggerDeliveryRecord } from '@librechat/data-schemas';
+import type { AgentTriggerEnvelope } from './envelope';
+import {
+ createAgentEventActorDetachedActionLifecycle,
+ createAgentEventDetachedResumeHandler,
+ isAgentEventActorDetachedActionProducerEnabled,
+} from './detachedAction';
+
+describe('isAgentEventActorDetachedActionProducerEnabled', () => {
+ const key = 'AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED';
+ const original = process.env[key];
+
+ afterEach(() => {
+ if (original == null) {
+ delete process.env[key];
+ return;
+ }
+ process.env[key] = original;
+ });
+
+ it('requires an explicit true activation', () => {
+ delete process.env[key];
+ expect(isAgentEventActorDetachedActionProducerEnabled()).toBe(false);
+ process.env[key] = 'false';
+ expect(isAgentEventActorDetachedActionProducerEnabled()).toBe(false);
+ process.env[key] = ' TRUE ';
+ expect(isAgentEventActorDetachedActionProducerEnabled()).toBe(true);
+ });
+});
+
+describe('createAgentEventActorDetachedActionLifecycle', () => {
+ it('owns only the exact expected action and exposes a suspension after launch', async () => {
+ const action = {
+ version: 1 as const,
+ invocationId: 'delivery-1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-1',
+ turnId: 'response-1:0',
+ taskId: `event_actor_${'a'.repeat(64)}`,
+ idempotencyKey: 'a'.repeat(64),
+ launchAttempt: 0 as const,
+ status: 'reserved' as const,
+ reservedAt: new Date('2026-08-28T12:00:00.000Z'),
+ observedAt: new Date('2026-08-28T12:00:00.000Z'),
+ recoveryAfter: new Date('2026-08-28T12:01:00.000Z'),
+ };
+ const reserve = jest.fn(async () => ({ status: 'reserved' as const, action }));
+ const markRunning = jest.fn(async () => ({ status: 'applied' as const }));
+ const settle = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('mongo unavailable'))
+ .mockResolvedValue({ status: 'applied' as const });
+ const waitForTerminalPersistenceRetry = jest.fn(async () => undefined);
+ const persistTerminalEvidence = jest.fn(async () => undefined);
+ const wake = jest.fn(async () => undefined);
+ const lifecycle = createAgentEventActorDetachedActionLifecycle(
+ {
+ user: 'user-1',
+ tenantId: 'tenant-1',
+ bindingId: 'binding-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 123,
+ turnCreatedAt: 456,
+ invocationId: 'delivery-1',
+ expectedAction: {
+ toolName: 'submit_move',
+ argumentSubset: { gameId: 'game-1' },
+ },
+ },
+ {
+ reserveAgentEventActorDetachedAction: reserve,
+ markAgentEventActorDetachedActionRunning: markRunning,
+ settleAgentEventActorDetachedAction: settle,
+ persistTerminalEvidence,
+ onTerminal: wake,
+ waitForTerminalPersistenceRetry,
+ producerEnabled: () => true,
+ now: () => new Date('2026-08-28T12:00:00.000Z'),
+ },
+ );
+
+ await expect(
+ lifecycle.reserve({
+ toolName: 'unrelated_tool',
+ toolCallId: 'call-0',
+ turnId: 'response-1:0',
+ arguments: { gameId: 'game-1' },
+ }),
+ ).resolves.toEqual({ status: 'ignored' });
+ await expect(
+ lifecycle.reserve({
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-1',
+ turnId: 'response-1:0',
+ arguments: { gameId: 'game-1', move: 'e4' },
+ }),
+ ).resolves.toEqual({
+ status: 'reserved',
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ });
+ expect(lifecycle.readSuspension()).toBeUndefined();
+
+ await expect(
+ lifecycle.markRunning({ taskId: action.taskId, idempotencyKey: action.idempotencyKey }),
+ ).resolves.toBe(true);
+
+ expect(lifecycle.readSuspension()).toEqual({
+ kind: 'internal_completion',
+ actionId: action.taskId,
+ jobCreatedAt: 456,
+ interrupt: {
+ id: action.taskId,
+ payload: {
+ type: 'event_actor_detached_action',
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ },
+ },
+ });
+ await expect(
+ lifecycle.settle({
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ status: 'succeeded',
+ result: { content: 'move accepted' },
+ }),
+ ).resolves.toBe(true);
+ await lifecycle.wake({ taskId: action.taskId, idempotencyKey: action.idempotencyKey });
+ expect(settle).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'succeeded', result: '{"content":"move accepted"}' }),
+ );
+ expect(settle).toHaveBeenCalledTimes(2);
+ expect(persistTerminalEvidence).toHaveBeenCalledTimes(1);
+ expect(persistTerminalEvidence).toHaveBeenCalledWith({
+ version: 1,
+ deliveryKey: 'delivery-1',
+ generationCreatedAt: 123,
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ status: 'succeeded',
+ result: '{"content":"move accepted"}',
+ observedAt: new Date('2026-08-28T12:00:00.000Z').getTime(),
+ });
+ expect(waitForTerminalPersistenceRetry).toHaveBeenCalledWith(100);
+ expect(reserve).toHaveBeenCalledWith(expect.objectContaining({ turnId: 'response-1:0' }));
+ expect(wake).toHaveBeenCalledWith({
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ });
+ });
+
+ it('fails closed before durable reservation until detached production is activated', async () => {
+ const reserve = jest.fn();
+ const lifecycle = createAgentEventActorDetachedActionLifecycle(
+ {
+ user: 'user-1',
+ bindingId: 'binding-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 123,
+ turnCreatedAt: 123,
+ invocationId: 'delivery-1',
+ expectedAction: { toolName: 'submit_move' },
+ },
+ {
+ reserveAgentEventActorDetachedAction: reserve,
+ markAgentEventActorDetachedActionRunning: jest.fn(),
+ settleAgentEventActorDetachedAction: jest.fn(),
+ persistTerminalEvidence: jest.fn(),
+ onTerminal: jest.fn(),
+ producerEnabled: () => false,
+ },
+ );
+
+ await expect(
+ lifecycle.reserve({
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-1',
+ turnId: 'response-1:0',
+ arguments: {},
+ }),
+ ).resolves.toEqual({
+ status: 'conflict',
+ error: expect.stringContaining('not activated'),
+ });
+ expect(reserve).not.toHaveBeenCalled();
+ expect(lifecycle.readSuspension()).toBeUndefined();
+ });
+
+ it('returns exact terminal evidence instead of projecting a stale running handle', async () => {
+ const action = {
+ version: 1 as const,
+ invocationId: 'delivery-1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-terminal',
+ turnId: 'response-1:0',
+ taskId: `event_actor_${'b'.repeat(64)}`,
+ idempotencyKey: 'b'.repeat(64),
+ launchAttempt: 1,
+ status: 'failed' as const,
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ settledAt: new Date(),
+ recoveryAfter: new Date(),
+ error: 'service unavailable',
+ };
+ const lifecycle = createAgentEventActorDetachedActionLifecycle(
+ {
+ user: 'user-1',
+ bindingId: 'binding-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 123,
+ turnCreatedAt: 123,
+ invocationId: 'delivery-1',
+ expectedAction: { toolName: 'submit_move' },
+ },
+ {
+ reserveAgentEventActorDetachedAction: jest.fn(async () => ({
+ status: 'replay' as const,
+ action,
+ })),
+ markAgentEventActorDetachedActionRunning: jest.fn(),
+ settleAgentEventActorDetachedAction: jest.fn(),
+ persistTerminalEvidence: jest.fn(),
+ onTerminal: jest.fn(),
+ producerEnabled: () => true,
+ },
+ );
+
+ await expect(
+ lifecycle.reserve({
+ toolName: action.toolName,
+ toolCallId: action.toolCallId,
+ turnId: 'response-1:0',
+ arguments: {},
+ }),
+ ).resolves.toEqual({
+ status: 'terminal',
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ outcome: 'failed',
+ error: action.error,
+ });
+ expect(lifecycle.readSuspension()).toBeUndefined();
+ });
+
+ it('stages terminal evidence durably before retrying the authoritative write', async () => {
+ const action = {
+ version: 1 as const,
+ invocationId: 'delivery-1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-outbox',
+ turnId: 'response-1:0',
+ taskId: `event_actor_${'e'.repeat(64)}`,
+ idempotencyKey: 'e'.repeat(64),
+ launchAttempt: 0,
+ status: 'reserved' as const,
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ };
+ const persistTerminalEvidence = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('job store unavailable'))
+ .mockResolvedValue(undefined);
+ const settle = jest.fn().mockResolvedValue({ status: 'applied' });
+ const waitForTerminalPersistenceRetry = jest.fn(async () => undefined);
+ const lifecycle = createAgentEventActorDetachedActionLifecycle(
+ {
+ user: 'user-1',
+ bindingId: 'binding-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 123,
+ turnCreatedAt: 123,
+ invocationId: 'delivery-1',
+ expectedAction: { toolName: 'submit_move' },
+ },
+ {
+ reserveAgentEventActorDetachedAction: jest.fn(async () => ({
+ status: 'reserved' as const,
+ action,
+ })),
+ markAgentEventActorDetachedActionRunning: jest.fn(async () => ({
+ status: 'applied' as const,
+ })),
+ settleAgentEventActorDetachedAction: settle,
+ persistTerminalEvidence,
+ onTerminal: jest.fn(),
+ waitForTerminalPersistenceRetry,
+ producerEnabled: () => true,
+ },
+ );
+ const reservation = await lifecycle.reserve({
+ toolName: action.toolName,
+ toolCallId: action.toolCallId,
+ turnId: 'response-1:0',
+ arguments: {},
+ });
+ expect(reservation.status).toBe('reserved');
+
+ await expect(
+ lifecycle.settle({
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ status: 'failed',
+ error: 'launch failed',
+ }),
+ ).resolves.toBe(true);
+
+ expect(persistTerminalEvidence).toHaveBeenCalledTimes(2);
+ expect(settle).toHaveBeenCalledTimes(1);
+ expect(persistTerminalEvidence.mock.invocationCallOrder[1]).toBeLessThan(
+ settle.mock.invocationCallOrder[0],
+ );
+ expect(waitForTerminalPersistenceRetry).toHaveBeenCalledWith(100);
+ });
+
+ it('preserves a same-executor replay of its running reservation', async () => {
+ const action = {
+ version: 1 as const,
+ invocationId: 'delivery-1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-replay',
+ turnId: 'response-1:0',
+ taskId: `event_actor_${'d'.repeat(64)}`,
+ idempotencyKey: 'd'.repeat(64),
+ launchAttempt: 0,
+ status: 'reserved' as const,
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ };
+ const reserve = jest
+ .fn()
+ .mockResolvedValueOnce({ status: 'reserved' as const, action })
+ .mockResolvedValueOnce({
+ status: 'replay' as const,
+ action: { ...action, status: 'running' as const },
+ });
+ const lifecycle = createAgentEventActorDetachedActionLifecycle(
+ {
+ user: 'user-1',
+ bindingId: 'binding-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 123,
+ turnCreatedAt: 123,
+ invocationId: 'delivery-1',
+ expectedAction: { toolName: 'submit_move' },
+ },
+ {
+ reserveAgentEventActorDetachedAction: reserve,
+ markAgentEventActorDetachedActionRunning: jest.fn(),
+ settleAgentEventActorDetachedAction: jest.fn(),
+ persistTerminalEvidence: jest.fn(),
+ onTerminal: jest.fn(),
+ producerEnabled: () => true,
+ },
+ );
+ const input = {
+ toolName: action.toolName,
+ toolCallId: action.toolCallId,
+ turnId: 'response-1:0',
+ arguments: {},
+ };
+
+ await expect(lifecycle.reserve(input)).resolves.toEqual({
+ status: 'reserved',
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ });
+ await expect(lifecycle.reserve(input)).resolves.toEqual({
+ status: 'replay',
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ });
+ });
+
+ it('refuses an indeterminate replay', async () => {
+ const baseAction = {
+ version: 1 as const,
+ invocationId: 'delivery-1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-release',
+ turnId: 'response-1:0',
+ taskId: `event_actor_${'c'.repeat(64)}`,
+ idempotencyKey: 'c'.repeat(64),
+ launchAttempt: 0,
+ status: 'reserved' as const,
+ reservedAt: new Date('2026-08-28T12:00:00.000Z'),
+ observedAt: new Date('2026-08-28T12:00:00.000Z'),
+ recoveryAfter: new Date('2026-08-28T12:01:00.000Z'),
+ };
+ const reserve = jest
+ .fn()
+ .mockResolvedValueOnce({ status: 'reserved' as const, action: baseAction })
+ .mockResolvedValueOnce({
+ status: 'replay' as const,
+ action: { ...baseAction, status: 'launch_indeterminate' as const },
+ });
+ const dependencies = {
+ reserveAgentEventActorDetachedAction: reserve,
+ markAgentEventActorDetachedActionRunning: jest.fn(),
+ settleAgentEventActorDetachedAction: jest.fn(),
+ persistTerminalEvidence: jest.fn(),
+ onTerminal: jest.fn(),
+ producerEnabled: () => true,
+ now: () => new Date('2026-08-28T12:00:00.000Z'),
+ };
+ const owner = {
+ user: 'user-1',
+ bindingId: 'binding-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 123,
+ turnCreatedAt: 123,
+ invocationId: 'delivery-1',
+ expectedAction: { toolName: 'submit_move' },
+ };
+ const first = createAgentEventActorDetachedActionLifecycle(owner, dependencies);
+ const reservation = await first.reserve({
+ toolName: baseAction.toolName,
+ toolCallId: baseAction.toolCallId,
+ turnId: 'response-1:0',
+ arguments: {},
+ });
+ expect(reservation.status).toBe('reserved');
+ expect(first.readSuspension()).toBeUndefined();
+
+ const recovered = createAgentEventActorDetachedActionLifecycle(owner, dependencies);
+ await expect(
+ recovered.reserve({
+ toolName: baseAction.toolName,
+ toolCallId: baseAction.toolCallId,
+ turnId: 'response-1:0',
+ arguments: {},
+ }),
+ ).resolves.toEqual(
+ expect.objectContaining({
+ status: 'conflict',
+ error: expect.stringContaining('indeterminate'),
+ }),
+ );
+ });
+});
+
+describe('createAgentEventDetachedResumeHandler', () => {
+ it('enqueues duplicate completion wakes under one stable mailbox identity', async () => {
+ const envelope = {
+ version: 1 as const,
+ mode: 'continue' as const,
+ requestId: 'request-original',
+ deliveryId: 'delivery-original',
+ receivedAt: 1,
+ principal: { userId: 'user-1' },
+ event: {
+ id: 'event-1',
+ type: 'game.move',
+ occurredAt: 1,
+ source: { id: 'source-1', type: 'remote_api_key' },
+ },
+ target: {
+ agentId: 'agent-1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'message-1',
+ bindingId: 'binding-1',
+ sourceKeyId: 'source-key-1',
+ },
+ input: 'play',
+ expectedAction: { toolName: 'submit_move' },
+ };
+ const delivery = {
+ id: 'row-1',
+ deliveryKey: 'trigger_original',
+ fingerprint: 'fingerprint',
+ orderingKey: 'lane',
+ laneSequence: 1,
+ envelope,
+ user: 'user-1',
+ status: 'succeeded',
+ attempts: 1,
+ availableAt: new Date(),
+ createdAt: new Date(),
+ } as unknown as AgentTriggerDeliveryRecord;
+ const enqueueAgentTrigger = jest.fn(async (_envelope: AgentTriggerEnvelope) => undefined);
+ let request = 0;
+ const resume = createAgentEventDetachedResumeHandler({
+ getAgentTriggerDelivery: jest.fn(async () => delivery),
+ enqueueAgentTrigger,
+ requestId: () => `wake-${++request}`,
+ now: () => 1_787_000_001_000,
+ });
+ const action = {
+ version: 1 as const,
+ invocationId: 'trigger_original',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-1',
+ turnId: 'response-1:0',
+ taskId: 'task-1',
+ idempotencyKey: 'a'.repeat(64),
+ launchAttempt: 0,
+ status: 'succeeded' as const,
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ settledAt: new Date(1_787_000_000_500),
+ result: 'accepted',
+ };
+ const input = {
+ streamId: 'conversation-1',
+ handlingGenerationCreatedAt: 1_786_999_999_000,
+ job: {
+ streamId: 'conversation-1',
+ conversationId: 'conversation-1',
+ userId: 'user-1',
+ status: 'complete' as const,
+ createdAt: 1_787_000_000_000,
+ syncSent: false,
+ agentEventDeliveryKey: 'trigger_original',
+ agentEventBindingId: 'binding-1',
+ },
+ suspension: {} as never,
+ action,
+ };
+
+ await resume(input);
+ await resume(input);
+
+ expect(enqueueAgentTrigger).toHaveBeenCalledTimes(2);
+ expect(enqueueAgentTrigger).toHaveBeenNthCalledWith(1, expect.any(Object), {
+ requiredWorkerCapability: 'event_actor_detached_action_v1',
+ });
+ const [first] = enqueueAgentTrigger.mock.calls[0];
+ const [second] = enqueueAgentTrigger.mock.calls[1];
+ expect(first.deliveryId).toBe('detached_completion:task-1');
+ expect(second.deliveryId).toBe(first.deliveryId);
+ expect(second.requestId).not.toBe(first.requestId);
+ expect(first.event.payload).toEqual(
+ expect.objectContaining({
+ invocationId: 'trigger_original',
+ generationCreatedAt: 1_786_999_999_000,
+ wakeGenerationCreatedAt: 1_787_000_000_000,
+ taskId: 'task-1',
+ }),
+ );
+ });
+});
diff --git a/packages/api/src/agents/triggers/detachedAction.ts b/packages/api/src/agents/triggers/detachedAction.ts
new file mode 100644
index 0000000000..1aee9e2ce5
--- /dev/null
+++ b/packages/api/src/agents/triggers/detachedAction.ts
@@ -0,0 +1,473 @@
+import { randomUUID } from 'node:crypto';
+import {
+ logger,
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+} from '@librechat/data-schemas';
+import type {
+ AgentEventActorDetachedAction,
+ AgentTriggerDeliveryMethods,
+ IAgentEventActorSuspensionEvidence,
+} from '@librechat/data-schemas';
+import type { EventActorInterrupt } from '@librechat/agents';
+import type {
+ AgentContinueTriggerEnvelope,
+ AgentTriggerEnvelope,
+ AgentTriggerExpectedAction,
+} from './envelope';
+import type { EventActorDetachedActionLifecycle } from '../handlers';
+import type { AgentEventDetachedTerminalEvidence } from './types';
+import type { SerializableJobData } from '~/stream';
+import { matchesExpectedAction } from './expectedAction';
+import { createAgentTriggerEnvelope } from './envelope';
+
+const MAX_TERMINAL_RESULT_LENGTH = 32_768;
+const TERMINAL_PERSIST_RETRY_INITIAL_MS = 100;
+const TERMINAL_PERSIST_RETRY_MAX_MS = 30_000;
+const RESERVATION_RECOVERY_MS = 60_000;
+const RUNNING_RECOVERY_MS = 30 * 60_000;
+const DETACHED_ACTION_PRODUCER_ENV = 'AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED';
+export const EVENT_ACTOR_DETACHED_COMPLETION_TYPE = 'librechat.event_actor.detached_completion';
+export const EVENT_ACTOR_DETACHED_COMPLETION_SOURCE = 'librechat-event-actor';
+
+export function isAgentEventActorDetachedActionProducerEnabled(): boolean {
+ return process.env[DETACHED_ACTION_PRODUCER_ENV]?.trim().toLowerCase() === 'true';
+}
+
+export function parseAgentEventDetachedTerminalEvidence(
+ value: unknown,
+): AgentEventDetachedTerminalEvidence | undefined {
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
+ return undefined;
+ }
+ const input = value as Record<string, unknown>;
+ const status = input.status;
+ if (
+ input.version !== 1 ||
+ typeof input.deliveryKey !== 'string' ||
+ input.deliveryKey.length === 0 ||
+ input.deliveryKey.length > 128 ||
+ !Number.isSafeInteger(input.generationCreatedAt) ||
+ (input.generationCreatedAt as number) < 0 ||
+ typeof input.taskId !== 'string' ||
+ input.taskId.length === 0 ||
+ input.taskId.length > 128 ||
+ typeof input.idempotencyKey !== 'string' ||
+ !/^[a-f0-9]{64}$/.test(input.idempotencyKey) ||
+ (status !== 'succeeded' && status !== 'failed' && status !== 'cancelled') ||
+ !Number.isSafeInteger(input.observedAt) ||
+ (input.observedAt as number) < 0 ||
+ (input.result != null &&
+ (typeof input.result !== 'string' || input.result.length > MAX_TERMINAL_RESULT_LENGTH)) ||
+ (input.error != null && (typeof input.error !== 'string' || input.error.length > 2_048)) ||
+ (status === 'succeeded'
+ ? typeof input.result !== 'string' || input.error != null
+ : typeof input.error !== 'string' || input.result != null)
+ ) {
+ return undefined;
+ }
+ return {
+ version: 1,
+ deliveryKey: input.deliveryKey,
+ generationCreatedAt: input.generationCreatedAt as number,
+ taskId: input.taskId,
+ idempotencyKey: input.idempotencyKey,
+ status,
+ ...(input.result == null ? {} : { result: input.result as string }),
+ ...(input.error == null ? {} : { error: input.error as string }),
+ observedAt: input.observedAt as number,
+ };
+}
+
+export interface AgentEventActorDetachedCompletionProjection {
+ version: 1;
+ invocationId: string;
+ /** Generation that owns the original delivery's detached-action record. */
+ generationCreatedAt: number;
+ /** Generation whose terminal handling enqueued this wake. */
+ wakeGenerationCreatedAt: number;
+ taskId: string;
+ idempotencyKey: string;
+}
+
+export function parseAgentEventActorDetachedCompletion(
+ value: unknown,
+): AgentEventActorDetachedCompletionProjection | undefined {
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
+ return undefined;
+ }
+ const input = value as Record<string, unknown>;
+ if (
+ input.version !== 1 ||
+ typeof input.invocationId !== 'string' ||
+ input.invocationId.length === 0 ||
+ input.invocationId.length > 128 ||
+ !Number.isSafeInteger(input.generationCreatedAt) ||
+ (input.generationCreatedAt as number) < 0 ||
+ !Number.isSafeInteger(input.wakeGenerationCreatedAt) ||
+ (input.wakeGenerationCreatedAt as number) < 0 ||
+ typeof input.taskId !== 'string' ||
+ input.taskId.length === 0 ||
+ input.taskId.length > 128 ||
+ typeof input.idempotencyKey !== 'string' ||
+ !/^[a-f0-9]{64}$/.test(input.idempotencyKey)
+ ) {
+ return undefined;
+ }
+ return {
+ version: 1,
+ invocationId: input.invocationId,
+ generationCreatedAt: input.generationCreatedAt as number,
+ wakeGenerationCreatedAt: input.wakeGenerationCreatedAt as number,
+ taskId: input.taskId,
+ idempotencyKey: input.idempotencyKey,
+ };
+}
+
+interface DetachedActionOwner {
+ user: string;
+ tenantId?: string;
+ bindingId: string;
+ conversationId: string;
+ generationCreatedAt: number;
+ turnCreatedAt: number;
+ invocationId: string;
+ expectedAction: AgentTriggerExpectedAction;
+}
+
+interface DetachedActionDependencies {
+ reserveAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['reserveAgentEventActorDetachedAction'];
+ markAgentEventActorDetachedActionRunning: AgentTriggerDeliveryMethods['markAgentEventActorDetachedActionRunning'];
+ settleAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['settleAgentEventActorDetachedAction'];
+ persistTerminalEvidence(input: AgentEventDetachedTerminalEvidence): Promise<void>;
+ onTerminal(input: { taskId: string; idempotencyKey: string }): Promise<void>;
+ waitForTerminalPersistenceRetry?(delayMs: number): Promise<void>;
+ producerEnabled(): boolean;
+ now?(): Date;
+}
+
+function waitForTerminalPersistenceRetry(delayMs: number): Promise<void> {
+ return new Promise((resolve) => {
+ const timer = setTimeout(resolve, delayMs);
+ timer.unref();
+ });
+}
+
+export interface AgentEventActorInternalSuspension {
+ kind: 'internal_completion';
+ actionId: string;
+ jobCreatedAt: number;
+ interrupt: EventActorInterrupt;
+}
+
+export interface AgentEventActorDetachedActionLifecycle extends EventActorDetachedActionLifecycle {
+ readSuspension(): AgentEventActorInternalSuspension | undefined;
+}
+
+export interface AgentEventActorDetachedResumeInput {
+ streamId: string;
+ job: SerializableJobData;
+ handlingGenerationCreatedAt: number;
+ suspension: IAgentEventActorSuspensionEvidence;
+ action: AgentEventActorDetachedAction;
+}
+
+interface DetachedResumeDependencies {
+ getAgentTriggerDelivery: AgentTriggerDeliveryMethods['getAgentTriggerDelivery'];
+ enqueueAgentTrigger(
+ envelope: AgentTriggerEnvelope,
+ options?: { requiredWorkerCapability?: string },
+ ): Promise<unknown>;
+ requestId?(): string;
+ now?(): number;
+}
+
+function renderDetachedCompletion(action: AgentEventActorDetachedAction): string {
+ const terminal =
+ action.status === 'succeeded'
+ ? `succeeded with this result:\n${action.result ?? ''}`
+ : `${action.status} with this error:\n${action.error ?? 'No error detail was recorded.'}`;
+ return [
+ 'A detached expected action from your preceding event turn has now reached an authoritative terminal state.',
+ `Tool: ${action.toolName}`,
+ `Tool call: ${action.toolCallId}`,
+ `The action ${terminal}`,
+ 'Continue the same event invocation using this trusted internal completion. Do not launch the same action again.',
+ ].join('\n');
+}
+
+/** Builds the exact internal continuation in the typed trigger layer. The app
+ * server supplies only persistence and dispatch composition dependencies. */
+export function createAgentEventDetachedResumeHandler(deps: DetachedResumeDependencies) {
+ const nextRequestId = deps.requestId ?? randomUUID;
+ const now = deps.now ?? Date.now;
+ return async ({
+ job,
+ action,
+ handlingGenerationCreatedAt,
+ }: AgentEventActorDetachedResumeInput): Promise<void> => {
+ const delivery = await deps.getAgentTriggerDelivery(job.agentEventDeliveryKey as string);
+ const envelope = delivery?.envelope as Partial<AgentContinueTriggerEnvelope> | undefined;
+ if (
+ delivery == null ||
+ String(delivery.user) !== job.userId ||
+ (delivery.tenantId ?? undefined) !== job.tenantId ||
+ envelope?.mode !== 'continue' ||
+ envelope.principal?.userId !== job.userId ||
+ (envelope.principal?.tenantId ?? undefined) !== job.tenantId ||
+ envelope.target?.bindingId !== job.agentEventBindingId ||
+ envelope.target?.conversationId !== job.conversationId ||
+ envelope.expectedAction?.toolName !== action.expectedToolName
+ ) {
+ throw new Error('Detached Event Actor completion owner is unavailable');
+ }
+ const target = envelope.target as AgentContinueTriggerEnvelope['target'];
+ const receivedAt = now();
+ const continuation = createAgentTriggerEnvelope({
+ mode: 'continue',
+ requestId: nextRequestId(),
+ deliveryId: `detached_completion:${action.taskId}`,
+ receivedAt,
+ principal: {
+ id: job.userId,
+ ...(job.tenantId == null ? {} : { tenantId: job.tenantId }),
+ },
+ event: {
+ id: action.taskId,
+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ occurredAt: action.settledAt?.getTime() ?? action.observedAt.getTime(),
+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },
+ payload: {
+ version: 1,
+ invocationId: job.agentEventDeliveryKey as string,
+ generationCreatedAt: handlingGenerationCreatedAt,
+ wakeGenerationCreatedAt: job.createdAt,
+ taskId: action.taskId,
+ idempotencyKey: action.idempotencyKey,
+ },
+ },
+ target,
+ input: renderDetachedCompletion(action),
+ expectedAction: envelope.expectedAction,
+ });
+ await deps.enqueueAgentTrigger(continuation, {
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ });
+ };
+}
+
+function serializeTerminalResult(value: unknown): string {
+ let serialized: string;
+ if (typeof value === 'string') {
+ serialized = value;
+ } else {
+ try {
+ serialized = JSON.stringify(value) ?? String(value);
+ } catch {
+ serialized = String(value);
+ }
+ }
+ return serialized.slice(0, MAX_TERMINAL_RESULT_LENGTH);
+}
+
+/**
+ * Binds the generic background executor to one delivery-owned expected action.
+ * The delivery row remains authoritative; request-local state only lets the
+ * actor adapter discover the exact launch that this segment already persisted.
+ */
+export function createAgentEventActorDetachedActionLifecycle(
+ owner: DetachedActionOwner,
+ deps: DetachedActionDependencies,
+): AgentEventActorDetachedActionLifecycle {
+ const now = deps.now ?? (() => new Date());
+ const producerEnabled = deps.producerEnabled();
+ let current: { taskId: string; idempotencyKey: string; launchAcknowledged: boolean } | undefined;
+ const scope = {
+ deliveryKey: owner.invocationId,
+ user: owner.user,
+ ...(owner.tenantId == null ? {} : { tenantId: owner.tenantId }),
+ bindingId: owner.bindingId,
+ conversationId: owner.conversationId,
+ generationCreatedAt: owner.generationCreatedAt,
+ };
+ const matchesCurrent = (input: { taskId: string; idempotencyKey: string }): boolean =>
+ current?.taskId === input.taskId && current.idempotencyKey === input.idempotencyKey;
+
+ return {
+ async reserve(input) {
+ if (
+ !matchesExpectedAction(
+ {
+ toolName: input.toolName,
+ toolCallId: input.toolCallId,
+ arguments: input.arguments,
+ },
+ owner.expectedAction,
+ )
+ ) {
+ return { status: 'ignored' };
+ }
+ if (!producerEnabled) {
+ return {
+ status: 'conflict',
+ error:
+ 'Detached Event Actor production is not activated for this deployment; no external action was launched',
+ };
+ }
+ const reservedAt = now();
+ const reservation = await deps.reserveAgentEventActorDetachedAction({
+ ...scope,
+ turnId: input.turnId,
+ invocationId: owner.invocationId,
+ expectedToolName: owner.expectedAction.toolName,
+ toolName: input.toolName,
+ toolCallId: input.toolCallId,
+ reservedAt,
+ recoveryAfter: new Date(reservedAt.getTime() + RESERVATION_RECOVERY_MS),
+ });
+ if (reservation.status === 'conflict') {
+ return {
+ status: 'conflict',
+ error: 'A different detached action already owns this Event Actor invocation',
+ };
+ }
+ if (
+ reservation.status === 'replay' &&
+ ['succeeded', 'failed', 'cancelled'].includes(reservation.action.status)
+ ) {
+ return {
+ status: 'terminal',
+ taskId: reservation.action.taskId,
+ idempotencyKey: reservation.action.idempotencyKey,
+ outcome: reservation.action.status as 'succeeded' | 'failed' | 'cancelled',
+ ...(reservation.action.result == null ? {} : { result: reservation.action.result }),
+ ...(reservation.action.error == null ? {} : { error: reservation.action.error }),
+ };
+ }
+ if (reservation.action.status === 'launch_indeterminate') {
+ return {
+ status: 'conflict',
+ error:
+ 'The detached Event Actor launch outcome is indeterminate; exact terminal proof is required before this invocation can continue',
+ };
+ }
+ if (reservation.status === 'replay') {
+ if (
+ matchesCurrent({
+ taskId: reservation.action.taskId,
+ idempotencyKey: reservation.action.idempotencyKey,
+ })
+ ) {
+ return {
+ status: 'replay',
+ taskId: reservation.action.taskId,
+ idempotencyKey: reservation.action.idempotencyKey,
+ };
+ }
+ return {
+ status: 'conflict',
+ error:
+ reservation.action.status === 'running'
+ ? 'The detached Event Actor action was already launched by another executor'
+ : 'The detached Event Actor launch acknowledgement is still pending',
+ };
+ }
+ current = {
+ taskId: reservation.action.taskId,
+ idempotencyKey: reservation.action.idempotencyKey,
+ launchAcknowledged: false,
+ };
+ return {
+ status: reservation.status,
+ taskId: current.taskId,
+ idempotencyKey: current.idempotencyKey,
+ };
+ },
+ async markRunning(input) {
+ if (!matchesCurrent(input)) {
+ return false;
+ }
+ const observedAt = now();
+ const marked = await deps.markAgentEventActorDetachedActionRunning({
+ ...scope,
+ ...input,
+ observedAt,
+ recoveryAfter: new Date(observedAt.getTime() + RUNNING_RECOVERY_MS),
+ });
+ const accepted = marked.status !== 'conflict';
+ if (accepted && current != null) {
+ current.launchAcknowledged = true;
+ }
+ return accepted;
+ },
+ async settle(input) {
+ if (!matchesCurrent(input)) {
+ return false;
+ }
+ const observedAt = now();
+ const evidence: AgentEventDetachedTerminalEvidence = {
+ version: 1,
+ deliveryKey: owner.invocationId,
+ generationCreatedAt: owner.generationCreatedAt,
+ taskId: input.taskId,
+ idempotencyKey: input.idempotencyKey,
+ status: input.status,
+ ...(input.status === 'succeeded'
+ ? { result: serializeTerminalResult(input.result) }
+ : { error: String(input.error ?? 'Detached action failed').slice(0, 2_048) }),
+ observedAt: observedAt.getTime(),
+ };
+ const waitForRetry = deps.waitForTerminalPersistenceRetry ?? waitForTerminalPersistenceRetry;
+ let retryDelayMs = TERMINAL_PERSIST_RETRY_INITIAL_MS;
+ let staged = false;
+ for (;;) {
+ try {
+ if (!staged) {
+ await deps.persistTerminalEvidence(evidence);
+ staged = true;
+ }
+ const settled = await deps.settleAgentEventActorDetachedAction({
+ ...scope,
+ taskId: evidence.taskId,
+ idempotencyKey: evidence.idempotencyKey,
+ status: evidence.status,
+ ...(evidence.result == null ? {} : { result: evidence.result }),
+ ...(evidence.error == null ? {} : { error: evidence.error }),
+ observedAt,
+ });
+ return settled.status !== 'conflict';
+ } catch (error) {
+ logger.warn('[event-actor] Retrying detached terminal evidence persistence', {
+ taskId: input.taskId,
+ retryDelayMs,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ await waitForRetry(retryDelayMs);
+ retryDelayMs = Math.min(retryDelayMs * 2, TERMINAL_PERSIST_RETRY_MAX_MS);
+ }
+ }
+ },
+ async wake(input) {
+ if (matchesCurrent(input)) {
+ await deps.onTerminal(input);
+ }
+ },
+ readSuspension() {
+ if (current?.launchAcknowledged !== true) {
+ return undefined;
+ }
+ return {
+ kind: 'internal_completion',
+ actionId: current.taskId,
+ jobCreatedAt: owner.turnCreatedAt,
+ interrupt: {
+ id: current.taskId,
+ payload: {
+ type: 'event_actor_detached_action',
+ taskId: current.taskId,
+ idempotencyKey: current.idempotencyKey,
+ },
+ },
+ };
+ },
+ };
+}
diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts
index 502702a6c9..d6c3ad19ca 100644
--- a/packages/api/src/agents/triggers/engine.ts
+++ b/packages/api/src/agents/triggers/engine.ts
@@ -61,10 +61,14 @@ export class AgentTriggerDeliveryDeferredError extends Error {
export type AgentTriggerDeliveryStatus =
| 'staging'
+ | 'capability_staging'
| 'batched'
| 'pending'
+ | 'capability_pending'
| 'leased'
+ | 'capability_leased'
| 'succeeded'
+ | 'capability_dead'
| 'dead';
export interface AgentTriggerDeliveryFailure {
diff --git a/packages/api/src/agents/triggers/expectedAction.ts b/packages/api/src/agents/triggers/expectedAction.ts
new file mode 100644
index 0000000000..0e1b66953a
--- /dev/null
+++ b/packages/api/src/agents/triggers/expectedAction.ts
@@ -0,0 +1,54 @@
+import type { AgentTriggerExpectedAction } from './envelope';
+
+export interface CompletedToolEvidence {
+ toolName: string;
+ toolCallId?: string;
+ arguments?: unknown;
+}
+
+export function parseAgentExpectedActionArguments(value: unknown): unknown {
+ if (typeof value !== 'string') {
+ return value;
+ }
+ try {
+ return JSON.parse(value) as unknown;
+ } catch {
+ return value;
+ }
+}
+
+function containsSubset(value: unknown, subset: unknown): boolean {
+ if (Array.isArray(subset)) {
+ return (
+ Array.isArray(value) &&
+ value.length === subset.length &&
+ subset.every((expected, index) => containsSubset(value[index], expected))
+ );
+ }
+ if (subset == null || typeof subset !== 'object') {
+ return Object.is(value, subset);
+ }
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
+ return false;
+ }
+ return Object.entries(subset).every(([key, expected]) =>
+ containsSubset((value as Record<string, unknown>)[key], expected),
+ );
+}
+
+export function matchesExpectedAction(
+ evidence: CompletedToolEvidence,
+ expected: AgentTriggerExpectedAction,
+): boolean {
+ const nameMatches =
+ evidence.toolName === expected.toolName ||
+ evidence.toolName.startsWith(`${expected.toolName}_mcp_`);
+ return (
+ nameMatches &&
+ (expected.argumentSubset == null ||
+ containsSubset(
+ parseAgentExpectedActionArguments(evidence.arguments),
+ expected.argumentSubset,
+ ))
+ );
+}
diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.ts
index 7f10674a41..706fa0056c 100644
--- a/packages/api/src/agents/triggers/host.spec.ts
+++ b/packages/api/src/agents/triggers/host.spec.ts
@@ -1,6 +1,10 @@
import { Constants, EModelEndpoint } from 'librechat-data-provider';
import { getRequestId, getTenantId, getUserId } from '@librechat/data-schemas';
import type { AgentTriggerExecutionHostDeps, AgentTriggerFetch } from './host';
+import {
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+} from './detachedAction';
import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './envelope';
import { AgentTriggerExecutionError, createAgentTriggerExecutionHost } from './host';
@@ -625,6 +629,57 @@ describe('createAgentTriggerExecutionHost continue adapter', () => {
).not.toHaveProperty('payload');
});
+ it('projects only the bounded internal detached-completion authority', async () => {
+ const base = createContinueEnvelope();
+ if (base.mode !== 'continue') {
+ throw new Error('Expected a continue envelope');
+ }
+ const completion = {
+ version: 1 as const,
+ invocationId: 'original-delivery-1',
+ generationCreatedAt: 1_787_000_000_000,
+ wakeGenerationCreatedAt: 1_787_000_000_000,
+ taskId: 'event-actor-task-1',
+ idempotencyKey: 'a'.repeat(64),
+ };
+ const envelope = {
+ ...base,
+ target: {
+ ...base.target,
+ bindingId: 'binding-1',
+ sourceKeyId: 'source-key-1',
+ },
+ event: {
+ id: completion.taskId,
+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ occurredAt: completion.generationCreatedAt + 1,
+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },
+ payload: completion,
+ },
+ expectedAction: { toolName: 'submit_move' },
+ };
+ const fetcher = fetchMock(async () =>
+ response({
+ streamId: 'conversation-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: completion.generationCreatedAt + 2,
+ status: 'started',
+ }),
+ );
+ const getBaseUrl = jest.fn(() => 'http://127.0.0.1:3080');
+
+ await createAgentTriggerExecutionHost(deps(fetcher, { getBaseUrl })).dispatch(envelope);
+
+ expect(getBaseUrl).toHaveBeenCalledWith({ localOnly: true });
+ const body = JSON.parse(String(fetcher.mock.calls[0][1]?.body));
+ expect(body.clientRequestId).toBe(getAgentTriggerIdempotencyKey(envelope));
+ expect(body.agentEventDelivery).toMatchObject({
+ deliveryKey: getAgentTriggerIdempotencyKey(envelope),
+ internalCompletion: completion,
+ });
+ expect(body.agentEventDelivery.event).not.toHaveProperty('payload');
+ });
+
it.each(['PARENT_NOT_READY', 'EVENT_ACTOR_NOT_READY'])(
'retries without consuming the logical delivery for temporary admission code %s',
async (code) => {
diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts
index a1477ab1b0..341041f27e 100644
--- a/packages/api/src/agents/triggers/host.ts
+++ b/packages/api/src/agents/triggers/host.ts
@@ -9,6 +9,11 @@ import type {
} from './envelope';
import type { AgentTriggerDispatchContext } from './dispatch';
import type { AgentRunPrincipal } from '../envelope';
+import {
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ parseAgentEventActorDetachedCompletion,
+} from './detachedAction';
import { dispatchAgentTrigger } from './dispatch';
const DEFAULT_FIRE_TIMEOUT_MS = 30_000;
@@ -119,7 +124,7 @@ export type AgentTriggerExecutionResult =
export interface AgentTriggerExecutionHostDeps {
/** Trusted root URL for this LibreChat server. */
- getBaseUrl: () => string;
+ getBaseUrl: (options?: { localOnly?: boolean }) => string;
/** Mint a short-lived token for the envelope's already-authenticated principal. */
mintToken: (principal: AgentRunPrincipal, envelope: AgentTriggerEnvelope) => MaybePromise<string>;
/** Optional user-timezone resolver for dynamic date variables in a new run. */
@@ -534,6 +539,13 @@ async function startRun(
timeoutMs: number,
): Promise<AgentTriggerContinueResult | AgentTriggerFireResult> {
const mode = envelope.mode;
+ const detachedCompletion =
+ mode === 'continue' &&
+ envelope.event.type === EVENT_ACTOR_DETACHED_COMPLETION_TYPE &&
+ envelope.event.source.type === 'internal' &&
+ envelope.event.source.id === EVENT_ACTOR_DETACHED_COMPLETION_SOURCE
+ ? parseAgentEventActorDetachedCompletion(envelope.event.payload)
+ : undefined;
const scope = abortScope(context.signal, timeoutMs);
let preparation: AgentTriggerContinuePreparation | undefined;
try {
@@ -574,7 +586,12 @@ async function startRun(
scope,
context.signal,
),
- setupValue(() => deps.getBaseUrl(), mode, scope, context.signal),
+ setupValue(
+ () => deps.getBaseUrl(detachedCompletion == null ? undefined : { localOnly: true }),
+ mode,
+ scope,
+ context.signal,
+ ),
]).catch((error: unknown) => {
scope.abort();
throw error;
@@ -632,6 +649,7 @@ async function startRun(
...(envelope.expectedAction != null && {
expectedAction: envelope.expectedAction,
}),
+ ...(detachedCompletion == null ? {} : { internalCompletion: detachedCompletion }),
},
}),
...(envelope.mode === 'fire' && {
diff --git a/packages/api/src/agents/triggers/index.ts b/packages/api/src/agents/triggers/index.ts
index 16b47f8d85..742e8ac79c 100644
--- a/packages/api/src/agents/triggers/index.ts
+++ b/packages/api/src/agents/triggers/index.ts
@@ -3,6 +3,7 @@ export * from './bindings';
export * from './actor';
export * from './bindingResolver';
export * from './delivery';
+export * from './detachedAction';
export * from './envelope';
export * from './host';
export * from './ingress';
diff --git a/packages/api/src/agents/triggers/outcome.spec.ts b/packages/api/src/agents/triggers/outcome.spec.ts
index e30e4e6890..89a2647b70 100644
--- a/packages/api/src/agents/triggers/outcome.spec.ts
+++ b/packages/api/src/agents/triggers/outcome.spec.ts
@@ -19,22 +19,32 @@ const createAgentEventTerminalHandler = (
'settleAgentTriggerHandlingOutcome'
> &
Partial<Parameters<typeof createAgentEventTerminalHandlerImpl>[0]>,
+ options?: Parameters<typeof createAgentEventTerminalHandlerImpl>[1],
) =>
- createAgentEventTerminalHandlerImpl({
- getAgentEventActorSnapshot: jest.fn().mockResolvedValue(undefined),
- getMessage: jest.fn().mockResolvedValue(null),
- resolveAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),
- clearAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),
- settleAgentEventActorReceipt: jest.fn().mockResolvedValue(true),
- getAgentEventActorReceipt: jest.fn().mockResolvedValue(null),
- backfillAgentEventActorReceipt: jest.fn().mockResolvedValue(true),
- completeAgentEventActorLegacyTurn: jest.fn().mockResolvedValue(true),
- cancelAgentEventActorSuspension: jest.fn().mockResolvedValue({ status: 'cancelled' }),
- releaseAgentEventActorAction: jest.fn().mockResolvedValue(true),
- getAgentEventActorActionAdmission: jest.fn().mockResolvedValue(null),
- hasAgentEventActorActionAdmission: jest.fn().mockResolvedValue(false),
- ...methods,
- });
+ createAgentEventTerminalHandlerImpl(
+ {
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue(undefined),
+ recordAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),
+ getMessage: jest.fn().mockResolvedValue(null),
+ resolveAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),
+ clearAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),
+ settleAgentEventActorReceipt: jest.fn().mockResolvedValue(true),
+ getAgentEventActorReceipt: jest.fn().mockResolvedValue(null),
+ backfillAgentEventActorReceipt: jest.fn().mockResolvedValue(true),
+ completeAgentEventActorLegacyTurn: jest.fn().mockResolvedValue(true),
+ cancelAgentEventActorSuspension: jest.fn().mockResolvedValue({ status: 'cancelled' }),
+ releaseAgentEventActorAction: jest.fn().mockResolvedValue(true),
+ getAgentEventActorActionAdmission: jest.fn().mockResolvedValue(null),
+ hasAgentEventActorActionAdmission: jest.fn().mockResolvedValue(false),
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(null),
+ settleAgentEventActorDetachedAction: jest.fn().mockResolvedValue({ status: 'applied' }),
+ markAgentEventActorDetachedActionLaunchIndeterminate: jest
+ .fn()
+ .mockResolvedValue({ status: 'applied' }),
+ ...methods,
+ },
+ options,
+ );
function job(overrides: Partial<SerializableJobData> = {}): SerializableJobData {
return {
@@ -107,6 +117,994 @@ describe('agent event terminal outcomes', () => {
mockedCancelAgentEventActor.mockResolvedValue({ status: 'cancelled' });
});
+ it('hands a terminal internal suspension to the durable resume adapter without settling', async () => {
+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
+ const resumeDetachedAction = jest.fn().mockResolvedValue(undefined);
+ const suspension = {
+ version: 1 as const,
+ suspensionId: 'suspension-detached-1',
+ attempt: 0,
+ issuedAt: 1,
+ expiresAt: 2,
+ invocation: {
+ actorThreadId: 'conversation-1',
+ invocationId: 'trigger_1',
+ depth: 0,
+ continuation: 'warm' as const,
+ base: { actorThreadId: 'conversation-1', generation: 1 },
+ fork: {
+ threadId: 'conversation-1',
+ checkpointNs: 'event-actor/trigger-1',
+ checkpointId: 'checkpoint-1',
+ invocationId: 'trigger_1',
+ },
+ },
+ checkpoint: {
+ threadId: 'conversation-1',
+ checkpointNs: 'event-actor/trigger-1',
+ checkpointId: 'checkpoint-1',
+ invocationId: 'trigger_1',
+ },
+ interrupt: { id: 'task-1', payload: { type: 'event_actor_detached_action' } },
+ suspensionDigest: 'signed-digest',
+ };
+ const detachedAction = {
+ version: 1 as const,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-1',
+ taskId: 'task-1',
+ idempotencyKey: 'a'.repeat(64),
+ launchAttempt: 0 as const,
+ status: 'succeeded' as const,
+ reservedAt: new Date(),
+ recoveryAfter: new Date(),
+ launchedAt: new Date(),
+ settledAt: new Date(),
+ observedAt: new Date(),
+ result: 'move accepted',
+ };
+ const settleAgentEventActorDetachedAction = jest
+ .fn()
+ .mockResolvedValue({ status: 'already_achieved' });
+ const handler = createAgentEventTerminalHandler(
+ {
+ settleAgentTriggerHandlingOutcome,
+ settleAgentEventActorDetachedAction,
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension,
+ actionId: 'task-1',
+ jobCreatedAt: 1_787_000_000_000,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ }),
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(detachedAction),
+ },
+ { resumeDetachedAction },
+ );
+
+ await handler(
+ 'conversation-1',
+ job({
+ agentEventBindingId: 'binding-1',
+ agentEventDetachedTerminalEvidence: {
+ version: 1,
+ deliveryKey: 'trigger_1',
+ generationCreatedAt: 1_787_000_000_000,
+ taskId: detachedAction.taskId,
+ idempotencyKey: detachedAction.idempotencyKey,
+ status: 'succeeded',
+ result: 'move accepted',
+ observedAt: 1_787_000_000_500,
+ },
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: 0,
+ },
+ }),
+ [],
+ );
+
+ expect(resumeDetachedAction).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: 'conversation-1', suspension, action: detachedAction }),
+ );
+ expect(settleAgentEventActorDetachedAction).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'trigger_1',
+ generationCreatedAt: 1_787_000_000_000,
+ taskId: detachedAction.taskId,
+ status: 'succeeded',
+ result: 'move accepted',
+ observedAt: new Date(1_787_000_000_500),
+ }),
+ );
+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();
+ });
+
+ it('durably enqueues a re-pause successor before settling its completion predecessor', async () => {
+ const events: string[] = [];
+ const suspension = suspensionEvidence('suspension-repause-successor', 1);
+ const action = {
+ version: 1 as const,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-successor',
+ taskId: 'task-successor',
+ idempotencyKey: '9'.repeat(64),
+ launchAttempt: 1,
+ status: 'succeeded' as const,
+ reservedAt: new Date(),
+ recoveryAfter: new Date(),
+ observedAt: new Date(),
+ settledAt: new Date(),
+ result: 'accepted',
+ };
+ const settleAgentTriggerHandlingOutcome = jest.fn(async () => {
+ events.push('settle-predecessor');
+ return true;
+ });
+ const resumeDetachedAction = jest.fn(async () => {
+ events.push('enqueue-successor');
+ });
+ const handler = createAgentEventTerminalHandler(
+ {
+ settleAgentTriggerHandlingOutcome,
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(action),
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension,
+ actionId: action.taskId,
+ jobCreatedAt: 1_787_000_010_000,
+ handlingGenerationCreatedAt: 1_787_000_000_000,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ }),
+ },
+ { resumeDetachedAction },
+ );
+
+ await handler(
+ 'conversation-1',
+ job({
+ createdAt: 1_787_000_010_000,
+ agentEventDeliveryKey: 'trigger_completion_1',
+ agentEventInvocationKey: 'trigger_1',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: suspension.attempt,
+ },
+ }),
+ [],
+ );
+
+ expect(events).toEqual(['enqueue-successor', 'settle-predecessor']);
+ expect(resumeDetachedAction).toHaveBeenCalledWith(
+ expect.objectContaining({ handlingGenerationCreatedAt: 1_787_000_000_000 }),
+ );
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'trigger_completion_1',
+ generationCreatedAt: 1_787_000_010_000,
+ status: 'completed_no_action',
+ }),
+ );
+ });
+
+ it('preserves original ownership and retires every predecessor across three completion hops', async () => {
+ const events: string[] = [];
+ const originalGenerationCreatedAt = 1_787_000_000_000;
+ const suspensions = [
+ suspensionEvidence('suspension-hop-1', 1),
+ suspensionEvidence('suspension-hop-2', 2),
+ ];
+ const actions = suspensions.map((_, index) => ({
+ version: 1 as const,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: `call-hop-${index + 1}`,
+ taskId: `task-hop-${index + 1}`,
+ idempotencyKey: String(index + 1).repeat(64),
+ launchAttempt: index + 1,
+ status: 'succeeded' as const,
+ reservedAt: new Date(),
+ recoveryAfter: new Date(),
+ observedAt: new Date(),
+ settledAt: new Date(),
+ result: 'accepted',
+ }));
+ const getAgentEventActorSnapshot = jest
+ .fn()
+ .mockResolvedValueOnce({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension: suspensions[0],
+ actionId: actions[0].taskId,
+ jobCreatedAt: originalGenerationCreatedAt + 10_000,
+ handlingGenerationCreatedAt: originalGenerationCreatedAt,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ })
+ .mockResolvedValueOnce({
+ state: null,
+ epoch: 2,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension: suspensions[1],
+ actionId: actions[1].taskId,
+ jobCreatedAt: originalGenerationCreatedAt + 20_000,
+ handlingGenerationCreatedAt: originalGenerationCreatedAt,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ });
+ const getAgentEventActorDetachedAction = jest
+ .fn()
+ .mockResolvedValueOnce(actions[0])
+ .mockResolvedValueOnce(actions[1]);
+ const settleAgentTriggerHandlingOutcome = jest.fn(async (input) => {
+ events.push(`settle:${input.deliveryKey}`);
+ return true;
+ });
+ const resumeDetachedAction = jest.fn(async (input) => {
+ events.push(`enqueue:${input.action.taskId}`);
+ });
+ const handler = createAgentEventTerminalHandler(
+ {
+ settleAgentTriggerHandlingOutcome,
+ getAgentEventActorSnapshot,
+ getAgentEventActorDetachedAction,
+ },
+ { resumeDetachedAction },
+ );
+
+ for (let index = 0; index < 2; index++) {
+ await handler(
+ 'conversation-1',
+ job({
+ createdAt: originalGenerationCreatedAt + (index + 1) * 10_000,
+ agentEventDeliveryKey: `trigger_completion_${index + 1}`,
+ agentEventInvocationKey: 'trigger_1',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspensions[index].suspensionId,
+ attempt: suspensions[index].attempt,
+ },
+ }),
+ [],
+ );
+ }
+
+ expect(events).toEqual([
+ 'enqueue:task-hop-1',
+ 'settle:trigger_completion_1',
+ 'enqueue:task-hop-2',
+ 'settle:trigger_completion_2',
+ ]);
+ expect(getAgentEventActorDetachedAction).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ generationCreatedAt: originalGenerationCreatedAt }),
+ );
+ expect(getAgentEventActorDetachedAction).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ generationCreatedAt: originalGenerationCreatedAt }),
+ );
+ expect(resumeDetachedAction).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ handlingGenerationCreatedAt: originalGenerationCreatedAt }),
+ );
+ });
+
+ it('repairs a crash after successor enqueue by replaying enqueue and predecessor settlement', async () => {
+ const suspension = suspensionEvidence('suspension-handoff-retry', 1);
+ const action = {
+ version: 1 as const,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-handoff-retry',
+ taskId: 'task-handoff-retry',
+ idempotencyKey: '6'.repeat(64),
+ launchAttempt: 1,
+ status: 'succeeded' as const,
+ reservedAt: new Date(),
+ recoveryAfter: new Date(),
+ observedAt: new Date(),
+ settledAt: new Date(),
+ result: 'accepted',
+ };
+ const snapshot = {
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion' as const,
+ suspension,
+ actionId: action.taskId,
+ jobCreatedAt: 1_787_000_010_000,
+ handlingGenerationCreatedAt: 1_787_000_000_000,
+ status: 'pending' as const,
+ observedAt: new Date(),
+ },
+ };
+ const resumeDetachedAction = jest.fn().mockResolvedValue(undefined);
+ const settleAgentTriggerHandlingOutcome = jest
+ .fn()
+ .mockResolvedValueOnce(false)
+ .mockResolvedValueOnce(true);
+ const handler = createAgentEventTerminalHandler(
+ {
+ settleAgentTriggerHandlingOutcome,
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(action),
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue(snapshot),
+ },
+ { resumeDetachedAction },
+ );
+ const completionJob = job({
+ createdAt: 1_787_000_010_000,
+ agentEventDeliveryKey: 'trigger_completion_retry',
+ agentEventInvocationKey: 'trigger_1',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: suspension.attempt,
+ },
+ });
+
+ await expect(handler('conversation-1', completionJob, [])).rejects.toThrow(
+ 'Failed to settle internal completion delivery',
+ );
+ await expect(handler('conversation-1', completionJob, [])).resolves.toBeUndefined();
+ expect(resumeDetachedAction).toHaveBeenCalledTimes(2);
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledTimes(2);
+ });
+
+ it('settles both the original invocation and its internal completion delivery', async () => {
+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
+ const getAgentEventActorDetachedAction = jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-failed',
+ taskId: 'task-failed',
+ idempotencyKey: 'b'.repeat(64),
+ launchAttempt: 0,
+ status: 'failed',
+ reservedAt: new Date(),
+ recoveryAfter: new Date(),
+ observedAt: new Date(),
+ settledAt: new Date(),
+ error: 'move service rejected the request',
+ });
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ getAgentEventActorDetachedAction,
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension: {
+ version: 1,
+ suspensionId: 'suspension-failed',
+ attempt: 0,
+ invocation: { invocationId: 'trigger_1' },
+ },
+ actionId: 'task-failed',
+ jobCreatedAt: 1_787_000_000_000,
+ handlingGenerationCreatedAt: 1_787_000_000_000,
+ status: 'closed',
+ outcome: 'settled',
+ observedAt: new Date(),
+ },
+ }),
+ });
+
+ await handler(
+ 'conversation-1',
+ job({
+ createdAt: 1_787_000_010_000,
+ agentEventDeliveryKey: 'trigger_completion_1',
+ agentEventInvocationKey: 'trigger_1',
+ agentEventBindingId: 'binding-1',
+ }),
+ [],
+ );
+
+ expect(getAgentEventActorDetachedAction).toHaveBeenCalledWith(
+ expect.objectContaining({ generationCreatedAt: 1_787_000_000_000 }),
+ );
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({
+ deliveryKey: 'trigger_1',
+ generationCreatedAt: 1_787_000_000_000,
+ status: 'failed',
+ error: 'move service rejected the request',
+ }),
+ );
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({
+ deliveryKey: 'trigger_completion_1',
+ generationCreatedAt: 1_787_000_010_000,
+ status: 'failed',
+ error: 'move service rejected the request',
+ }),
+ );
+ });
+
+ it('marks an expired detached launch indeterminate without resuming or relaunching', async () => {
+ const suspension = suspensionEvidence('suspension-indeterminate');
+ const getAgentEventActorDetachedAction = jest
+ .fn()
+ .mockResolvedValueOnce({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-indeterminate',
+ taskId: 'task-indeterminate',
+ idempotencyKey: 'c'.repeat(64),
+ launchAttempt: 0,
+ status: 'running',
+ reservedAt: new Date(0),
+ observedAt: new Date(0),
+ recoveryAfter: new Date(1),
+ })
+ .mockResolvedValueOnce({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-indeterminate',
+ taskId: 'task-indeterminate',
+ idempotencyKey: 'c'.repeat(64),
+ launchAttempt: 0,
+ status: 'launch_indeterminate',
+ reservedAt: new Date(0),
+ observedAt: new Date(),
+ recoveryAfter: new Date(1),
+ });
+ const markIndeterminate = jest.fn().mockResolvedValue({ status: 'applied' });
+ const resumeDetachedAction = jest.fn();
+ const handler = createAgentEventTerminalHandler(
+ {
+ settleAgentTriggerHandlingOutcome: jest.fn(),
+ getAgentEventActorDetachedAction,
+ markAgentEventActorDetachedActionLaunchIndeterminate: markIndeterminate,
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension,
+ actionId: 'task-indeterminate',
+ jobCreatedAt: 1_787_000_000_000,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ }),
+ },
+ { resumeDetachedAction },
+ );
+
+ await expect(
+ handler(
+ 'conversation-1',
+ job({
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: suspension.attempt,
+ },
+ }),
+ [],
+ ),
+ ).rejects.toThrow('detached action launch is indeterminate');
+ expect(markIndeterminate).toHaveBeenCalledTimes(1);
+ expect(resumeDetachedAction).not.toHaveBeenCalled();
+ });
+
+ it('keeps an indeterminate action owner open when the executor dies before suspension', async () => {
+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
+ const markIndeterminate = jest.fn().mockResolvedValue({ status: 'applied' });
+ const action = {
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-unacknowledged',
+ taskId: 'task-unacknowledged',
+ idempotencyKey: 'e'.repeat(64),
+ launchAttempt: 0,
+ status: 'running',
+ reservedAt: new Date(0),
+ observedAt: new Date(0),
+ recoveryAfter: new Date(1),
+ };
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ markAgentEventActorDetachedActionLaunchIndeterminate: markIndeterminate,
+ getAgentEventActorDetachedAction: jest
+ .fn()
+ .mockResolvedValueOnce(action)
+ .mockResolvedValueOnce({ ...action, status: 'launch_indeterminate' }),
+ });
+
+ await expect(
+ handler(
+ 'conversation-1',
+ job({ status: 'error', error: 'executor exited', agentEventBindingId: 'binding-1' }),
+ [],
+ ),
+ ).rejects.toThrow('detached action launch is indeterminate');
+
+ expect(markIndeterminate).toHaveBeenCalledTimes(1);
+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();
+ });
+
+ it('keeps an unexpired detached launch open after generation failure', async () => {
+ const settleAgentTriggerHandlingOutcome = jest.fn();
+ const markIndeterminate = jest.fn();
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ markAgentEventActorDetachedActionLaunchIndeterminate: markIndeterminate,
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-running',
+ taskId: 'task-running',
+ idempotencyKey: '8'.repeat(64),
+ launchAttempt: 0,
+ status: 'running',
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(Date.now() + 60_000),
+ }),
+ });
+
+ await expect(
+ handler(
+ 'conversation-1',
+ job({ status: 'error', error: 'generation failed', agentEventBindingId: 'binding-1' }),
+ [],
+ ),
+ ).rejects.toThrow('detached action is still in flight');
+ expect(markIndeterminate).not.toHaveBeenCalled();
+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();
+ });
+
+ it('cancels an aborted internal completion instead of waking it', async () => {
+ const suspension = suspensionEvidence('suspension-aborted-internal');
+ const resumeDetachedAction = jest.fn();
+ const handler = createAgentEventTerminalHandler(
+ {
+ settleAgentTriggerHandlingOutcome: jest.fn().mockResolvedValue(true),
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-aborted',
+ taskId: 'task-aborted',
+ idempotencyKey: 'd'.repeat(64),
+ launchAttempt: 0,
+ status: 'failed',
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ settledAt: new Date(),
+ error: 'detached action failed',
+ }),
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension,
+ actionId: 'task-aborted',
+ jobCreatedAt: 1_787_000_000_000,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ }),
+ },
+ { resumeDetachedAction },
+ );
+
+ await handler(
+ 'conversation-1',
+ job({
+ status: 'aborted',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: suspension.attempt,
+ },
+ }),
+ [],
+ );
+
+ expect(resumeDetachedAction).not.toHaveBeenCalled();
+ expect(mockedCancelAgentEventActor).toHaveBeenCalledTimes(1);
+ });
+
+ it('retires a failed detached action that became terminal before suspension storage', async () => {
+ const checkpoint = {
+ threadId: 'conversation-1',
+ checkpointNs: 'event-actor/trigger_1',
+ };
+ const initialSnapshot = {
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [
+ {
+ invocationId: 'trigger_1',
+ actionAdmitted: true,
+ status: 'invocation_pending' as const,
+ checkpoint,
+ action: { toolName: 'submit_move' },
+ observedAt: new Date(),
+ },
+ ],
+ };
+ const retiredSnapshot = { ...initialSnapshot, reconciliations: [] };
+ const getAgentEventActorSnapshot = jest
+ .fn()
+ .mockResolvedValueOnce(initialSnapshot)
+ .mockResolvedValue(retiredSnapshot);
+ const resolveAgentEventActorReconciliation = jest.fn().mockResolvedValue(true);
+ const getAgentEventActorActionAdmission = jest
+ .fn()
+ .mockResolvedValueOnce('admission-pre-suspension')
+ .mockResolvedValue(null);
+ const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true);
+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ getAgentEventActorSnapshot,
+ resolveAgentEventActorReconciliation,
+ getAgentEventActorActionAdmission,
+ releaseAgentEventActorAction,
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-pre-suspension-failure',
+ taskId: 'task-pre-suspension-failure',
+ idempotencyKey: 'f'.repeat(64),
+ launchAttempt: 0,
+ status: 'failed',
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ settledAt: new Date(),
+ error: 'detached action failed before suspension storage',
+ }),
+ });
+ const terminalJob = job({ agentEventBindingId: 'binding-1' });
+
+ await handler('conversation-1', terminalJob, []);
+ await handler('conversation-1', terminalJob, []);
+
+ expect(resolveAgentEventActorReconciliation).toHaveBeenCalledTimes(1);
+ expect(resolveAgentEventActorReconciliation).toHaveBeenCalledWith({
+ user: 'user-1',
+ conversationId: 'conversation-1',
+ invocationId: 'trigger_1',
+ checkpoint,
+ expectedActionAdmitted: true,
+ resolution: 'invocation_abandoned',
+ });
+ expect(releaseAgentEventActorAction).toHaveBeenCalledTimes(1);
+ expect(releaseAgentEventActorAction).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'trigger_1',
+ bindingId: 'binding-1',
+ admissionId: 'admission-pre-suspension',
+ }),
+ );
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledTimes(2);
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'trigger_1',
+ status: 'failed',
+ error: 'detached action failed before suspension storage',
+ }),
+ );
+ });
+
+ it('retires the claimed predecessor when a successor fails before re-pause storage', async () => {
+ const predecessor = suspensionEvidence('suspension-claimed-predecessor', 1);
+ const initialSnapshot = {
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [
+ {
+ invocationId: 'trigger_1',
+ actionAdmitted: true,
+ status: 'invocation_pending' as const,
+ checkpoint: predecessor.checkpoint,
+ action: { toolName: 'submit_move' },
+ observedAt: new Date(),
+ },
+ ],
+ suspension: {
+ kind: 'internal_completion' as const,
+ suspension: predecessor,
+ actionId: 'task-predecessor',
+ jobCreatedAt: 1_787_000_000_000,
+ handlingGenerationCreatedAt: 1_787_000_000_000,
+ status: 'claimed' as const,
+ resumeAttemptId: 'trigger_completion_failed_successor',
+ observedAt: new Date(),
+ },
+ };
+ const closedSnapshot = {
+ ...initialSnapshot,
+ reconciliations: [],
+ suspension: {
+ ...initialSnapshot.suspension,
+ status: 'closed' as const,
+ outcome: 'cancelled' as const,
+ },
+ };
+ const getAgentEventActorSnapshot = jest
+ .fn()
+ .mockResolvedValueOnce(initialSnapshot)
+ .mockResolvedValue(closedSnapshot);
+ const releaseAgentEventActorAction = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('delivery store unavailable'))
+ .mockResolvedValue(true);
+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ getAgentEventActorSnapshot,
+ releaseAgentEventActorAction,
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-failed-successor',
+ taskId: 'task-failed-successor',
+ idempotencyKey: '6'.repeat(64),
+ launchAttempt: 1,
+ status: 'failed',
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ settledAt: new Date(),
+ error: 'successor failed before re-pause storage',
+ }),
+ });
+
+ const terminalJob = job({
+ createdAt: 1_787_000_010_000,
+ agentEventDeliveryKey: 'trigger_completion_failed_successor',
+ agentEventInvocationKey: 'trigger_1',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: undefined,
+ });
+
+ await expect(handler('conversation-1', terminalJob, [])).rejects.toThrow(
+ 'delivery store unavailable',
+ );
+ await handler('conversation-1', terminalJob, []);
+
+ expect(mockedCancelAgentEventActor).toHaveBeenCalledWith(
+ expect.objectContaining({
+ suspension: predecessor,
+ claimedResumeAttemptId: 'trigger_completion_failed_successor',
+ reason: 'cancelled',
+ }),
+ expect.any(Object),
+ );
+ expect(releaseAgentEventActorAction).toHaveBeenCalledTimes(2);
+ expect(releaseAgentEventActorAction).toHaveBeenLastCalledWith(
+ expect.objectContaining({ deliveryKey: 'trigger_1', bindingId: 'binding-1' }),
+ );
+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledTimes(2);
+ expect(
+ settleAgentTriggerHandlingOutcome.mock.calls.map(([input]) => input.deliveryKey),
+ ).toEqual(['trigger_1', 'trigger_completion_failed_successor']);
+ expect(settleAgentTriggerHandlingOutcome.mock.calls[0][0]).toMatchObject({
+ status: 'failed',
+ error: 'successor failed before re-pause storage',
+ });
+ });
+
+ it('releases admission after an exact completion predecessor settled normally', async () => {
+ const predecessor = suspensionEvidence('suspension-settled-predecessor', 1);
+ const completionDeliveryKey = 'trigger_completion_settled_successor';
+ const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true);
+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ releaseAgentEventActorAction,
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [],
+ suspension: {
+ kind: 'internal_completion',
+ suspension: predecessor,
+ actionId: 'task-settled-predecessor',
+ jobCreatedAt: 1_787_000_000_000,
+ handlingGenerationCreatedAt: 1_787_000_000_000,
+ status: 'closed',
+ outcome: 'settled',
+ resumeAttemptId: completionDeliveryKey,
+ observedAt: new Date(),
+ },
+ }),
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-settled-successor',
+ turnId: 'response-settled-successor:0',
+ taskId: 'task-settled-successor',
+ idempotencyKey: '8'.repeat(64),
+ launchAttempt: 1,
+ status: 'succeeded',
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ settledAt: new Date(),
+ result: 'accepted',
+ }),
+ });
+
+ await handler(
+ 'conversation-1',
+ job({
+ createdAt: 1_787_000_010_000,
+ agentEventDeliveryKey: completionDeliveryKey,
+ agentEventInvocationKey: 'trigger_1',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: undefined,
+ }),
+ [],
+ );
+
+ expect(releaseAgentEventActorAction).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: 'trigger_1',
+ bindingId: 'binding-1',
+ admissionId: expect.any(String),
+ }),
+ );
+ expect(
+ settleAgentTriggerHandlingOutcome.mock.calls.map(([input]) => input.deliveryKey),
+ ).toEqual(['trigger_1', completionDeliveryKey]);
+ });
+
+ it('records reconciliation when a detached action succeeds after generation abort', async () => {
+ const suspension = suspensionEvidence('suspension-aborted-success');
+ const recordAgentEventActorReconciliation = jest.fn().mockResolvedValue(true);
+ const settleAgentTriggerHandlingOutcome = jest.fn();
+ const handler = createAgentEventTerminalHandler({
+ settleAgentTriggerHandlingOutcome,
+ recordAgentEventActorReconciliation,
+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({
+ version: 1,
+ invocationId: 'trigger_1',
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-aborted-success',
+ taskId: 'task-aborted-success',
+ idempotencyKey: '7'.repeat(64),
+ launchAttempt: 0,
+ status: 'succeeded',
+ reservedAt: new Date(),
+ observedAt: new Date(),
+ recoveryAfter: new Date(),
+ settledAt: new Date(),
+ result: 'accepted',
+ }),
+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({
+ state: null,
+ epoch: 1,
+ legacyTurn: null,
+ reconciliations: [
+ {
+ invocationId: 'trigger_1',
+ actionAdmitted: true,
+ status: 'invocation_pending',
+ checkpoint: suspension.checkpoint,
+ action: { toolName: 'submit_move' },
+ observedAt: new Date(),
+ },
+ ],
+ suspension: {
+ kind: 'internal_completion',
+ suspension,
+ actionId: 'task-aborted-success',
+ jobCreatedAt: job().createdAt,
+ status: 'pending',
+ observedAt: new Date(),
+ },
+ }),
+ });
+
+ await expect(
+ handler(
+ 'conversation-1',
+ job({
+ status: 'aborted',
+ agentEventBindingId: 'binding-1',
+ agentEventSuspension: {
+ version: 1,
+ suspensionId: suspension.suspensionId,
+ attempt: suspension.attempt,
+ },
+ }),
+ [],
+ ),
+ ).rejects.toThrow('requires commit_indeterminate reconciliation');
+ expect(recordAgentEventActorReconciliation).toHaveBeenCalledWith(
+ expect.objectContaining({
+ reconciliation: expect.objectContaining({
+ invocationId: 'trigger_1',
+ status: 'commit_indeterminate',
+ action: {
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-aborted-success',
+ },
+ }),
+ }),
+ );
+ expect(mockedCancelAgentEventActor).not.toHaveBeenCalled();
+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();
+ });
+
it('cancels a versioned paused actor before settling its expired delivery', async () => {
const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);
const suspension = {
diff --git a/packages/api/src/agents/triggers/outcome.ts b/packages/api/src/agents/triggers/outcome.ts
index 199625765c..387d93bcf8 100644
--- a/packages/api/src/agents/triggers/outcome.ts
+++ b/packages/api/src/agents/triggers/outcome.ts
@@ -1,16 +1,23 @@
import type {
+ AgentEventActorDetachedAction,
AgentTriggerDeliveryMethods,
ConversationMethods,
IAgentEventActorSuspension,
MessageMethods,
} from '@librechat/data-schemas';
import type { Agents } from 'librechat-data-provider';
+import type { AgentEventActorDetachedResumeInput } from './detachedAction';
+import type { CompletedToolEvidence } from './expectedAction';
import type { AgentTriggerExpectedAction } from './envelope';
import type { AgentEventAppliedAction } from './types';
import type { SerializableJobData } from '~/stream';
+import { matchesExpectedAction, parseAgentExpectedActionArguments } from './expectedAction';
import { cancelAgentEventActor, createAgentEventActorActionAdmissionId } from './actor';
+import { parseAgentEventDetachedTerminalEvidence } from './detachedAction';
export type { AgentEventAppliedAction } from './types';
+export { matchesExpectedAction } from './expectedAction';
+export type { CompletedToolEvidence } from './expectedAction';
interface SettleAgentTriggerHandlingOutcomeInput {
deliveryKey: string;
@@ -22,12 +29,6 @@ interface SettleAgentTriggerHandlingOutcomeInput {
action?: { toolName: string; toolCallId?: string };
}
-interface CompletedToolEvidence {
- toolName: string;
- toolCallId?: string;
- arguments?: unknown;
-}
-
export interface AgentEventRunOutcome {
status: 'applied' | 'completed_no_action' | 'failed' | 'cancelled';
action?: { toolName: string; toolCallId?: string };
@@ -55,7 +56,7 @@ async function hasDurableAgentEventHistory(input: {
}
function isBackgroundNonExecutionReceipt(value: unknown, argumentsValue: unknown): boolean {
- const parsedArguments = parseArguments(argumentsValue);
+ const parsedArguments = parseAgentExpectedActionArguments(argumentsValue);
if (
parsedArguments == null ||
typeof parsedArguments !== 'object' ||
@@ -64,7 +65,7 @@ function isBackgroundNonExecutionReceipt(value: unknown, argumentsValue: unknown
) {
return false;
}
- const parsed = parseArguments(value);
+ const parsed = parseAgentExpectedActionArguments(value);
if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
return false;
}
@@ -152,17 +153,6 @@ function nonExecutedHITLToolCallIds(
return ids;
}
-function parseArguments(value: unknown): unknown {
- if (typeof value !== 'string') {
- return value;
- }
- try {
- return JSON.parse(value) as unknown;
- } catch {
- return value;
- }
-}
-
/** Classifies terminal run evidence once for both checkpoint commit and public receipt. */
export function classifyAgentEventRunOutcome(
job: SerializableJobData,
@@ -206,39 +196,6 @@ export function findAgentEventAppliedAction(
};
}
-function containsSubset(value: unknown, subset: unknown): boolean {
- if (Array.isArray(subset)) {
- return (
- Array.isArray(value) &&
- value.length === subset.length &&
- subset.every((expected, index) => containsSubset(value[index], expected))
- );
- }
- if (subset == null || typeof subset !== 'object') {
- return Object.is(value, subset);
- }
- if (value == null || typeof value !== 'object' || Array.isArray(value)) {
- return false;
- }
- return Object.entries(subset).every(([key, expected]) =>
- containsSubset((value as Record<string, unknown>)[key], expected),
- );
-}
-
-function matchesExpectedAction(
- evidence: CompletedToolEvidence,
- expected: AgentTriggerExpectedAction,
-): boolean {
- const nameMatches =
- evidence.toolName === expected.toolName ||
- evidence.toolName.startsWith(`${expected.toolName}_mcp_`);
- return (
- nameMatches &&
- (expected.argumentSubset == null ||
- containsSubset(parseArguments(evidence.arguments), expected.argumentSubset))
- );
-}
-
export interface AgentEventActionRecorder {
observeToolEnd(data: {
input?: unknown;
@@ -281,7 +238,7 @@ export function createAgentEventActionRecorder(
* would be indistinguishable from a real result, so a call the model
* detached can never qualify through this shape. */
if (data.outputFiltered === true) {
- const parsedInput = parseArguments(data.input);
+ const parsedInput = parseAgentExpectedActionArguments(data.input);
if (
parsedInput != null &&
typeof parsedInput === 'object' &&
@@ -332,23 +289,32 @@ export function createAgentEventActionRecorder(
};
}
-export function createAgentEventTerminalHandler(methods: {
- settleAgentTriggerHandlingOutcome: (
- input: SettleAgentTriggerHandlingOutcomeInput,
- ) => Promise<boolean>;
- getAgentEventActorSnapshot: ConversationMethods['getAgentEventActorSnapshot'];
- resolveAgentEventActorReconciliation: ConversationMethods['resolveAgentEventActorReconciliation'];
- clearAgentEventActorReconciliation: ConversationMethods['clearAgentEventActorReconciliation'];
- settleAgentEventActorReceipt: AgentTriggerDeliveryMethods['settleAgentEventActorReceipt'];
- getAgentEventActorReceipt: AgentTriggerDeliveryMethods['getAgentEventActorReceipt'];
- backfillAgentEventActorReceipt: AgentTriggerDeliveryMethods['backfillAgentEventActorReceipt'];
- completeAgentEventActorLegacyTurn: ConversationMethods['completeAgentEventActorLegacyTurn'];
- cancelAgentEventActorSuspension: ConversationMethods['cancelAgentEventActorSuspension'];
- releaseAgentEventActorAction: AgentTriggerDeliveryMethods['releaseAgentEventActorAction'];
- getAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['getAgentEventActorActionAdmission'];
- hasAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['hasAgentEventActorActionAdmission'];
- getMessage: MessageMethods['getMessage'];
-}): (
+export function createAgentEventTerminalHandler(
+ methods: {
+ settleAgentTriggerHandlingOutcome: (
+ input: SettleAgentTriggerHandlingOutcomeInput,
+ ) => Promise<boolean>;
+ getAgentEventActorSnapshot: ConversationMethods['getAgentEventActorSnapshot'];
+ recordAgentEventActorReconciliation: ConversationMethods['recordAgentEventActorReconciliation'];
+ resolveAgentEventActorReconciliation: ConversationMethods['resolveAgentEventActorReconciliation'];
+ clearAgentEventActorReconciliation: ConversationMethods['clearAgentEventActorReconciliation'];
+ settleAgentEventActorReceipt: AgentTriggerDeliveryMethods['settleAgentEventActorReceipt'];
+ getAgentEventActorReceipt: AgentTriggerDeliveryMethods['getAgentEventActorReceipt'];
+ backfillAgentEventActorReceipt: AgentTriggerDeliveryMethods['backfillAgentEventActorReceipt'];
+ completeAgentEventActorLegacyTurn: ConversationMethods['completeAgentEventActorLegacyTurn'];
+ cancelAgentEventActorSuspension: ConversationMethods['cancelAgentEventActorSuspension'];
+ releaseAgentEventActorAction: AgentTriggerDeliveryMethods['releaseAgentEventActorAction'];
+ getAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['getAgentEventActorActionAdmission'];
+ hasAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['hasAgentEventActorActionAdmission'];
+ getAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['getAgentEventActorDetachedAction'];
+ settleAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['settleAgentEventActorDetachedAction'];
+ markAgentEventActorDetachedActionLaunchIndeterminate: AgentTriggerDeliveryMethods['markAgentEventActorDetachedActionLaunchIndeterminate'];
+ getMessage: MessageMethods['getMessage'];
+ },
+ options: {
+ resumeDetachedAction?(input: AgentEventActorDetachedResumeInput): Promise<void>;
+ } = {},
+): (
streamId: string,
job: SerializableJobData,
runSteps: Agents.RunStep[],
@@ -356,17 +322,50 @@ export function createAgentEventTerminalHandler(methods: {
) => Promise<void> {
return async (
streamId: string,
- job: SerializableJobData,
+ receivedJob: SerializableJobData,
runSteps: Agents.RunStep[],
content: Agents.MessageContentComplex[] = [],
) => {
- if (job.agentEventDeliveryKey == null) {
+ if (receivedJob.agentEventDeliveryKey == null) {
return;
}
+ const completionDeliveryKey =
+ receivedJob.agentEventInvocationKey == null ? undefined : receivedJob.agentEventDeliveryKey;
+ /** Actor state remains owned by the original invocation. The internal
+ * completion delivery separately owns this generation's mailbox lane. */
+ const job: SerializableJobData & { agentEventDeliveryKey: string } =
+ receivedJob.agentEventInvocationKey == null
+ ? { ...receivedJob, agentEventDeliveryKey: receivedJob.agentEventDeliveryKey }
+ : {
+ ...receivedJob,
+ agentEventDeliveryKey: receivedJob.agentEventInvocationKey,
+ };
const conversationId = job.conversationId ?? streamId;
const outcome = classifyAgentEventRunOutcome(job, runSteps, content);
const settledAt = new Date(job.completedAt ?? Date.now());
+ const settleCompletionDelivery = async (
+ completionOutcome: AgentEventRunOutcome,
+ failureError = receivedJob.error ?? 'Generation failed',
+ ): Promise<void> => {
+ if (completionDeliveryKey == null) {
+ return;
+ }
+ const completionSettled = await methods.settleAgentTriggerHandlingOutcome({
+ deliveryKey: completionDeliveryKey,
+ conversationId,
+ generationCreatedAt: receivedJob.createdAt,
+ status: completionOutcome.status,
+ settledAt,
+ ...(completionOutcome.status === 'failed' && { error: failureError }),
+ ...(completionOutcome.action != null && { action: completionOutcome.action }),
+ });
+ if (!completionSettled) {
+ throw new Error(`Failed to settle internal completion delivery ${completionDeliveryKey}`);
+ }
+ };
let committedAction: AgentEventAppliedAction | undefined;
+ let detachedTerminalFailure: string | undefined;
+ let detachedTerminalRetiredWithoutSuspension = false;
let compensated = false;
let actorReceiptSettled = false;
const owner = {
@@ -378,6 +377,255 @@ export function createAgentEventTerminalHandler(methods: {
let retiredWithoutAction: IAgentEventActorSuspension | undefined;
const isIrrecoverablyTerminal = job.status === 'aborted' || job.status === 'error';
const unprojectedSuspension = snapshot?.suspension;
+ const handlingGenerationCreatedAt =
+ unprojectedSuspension?.suspension.invocation.invocationId === job.agentEventDeliveryKey
+ ? (unprojectedSuspension.handlingGenerationCreatedAt ?? job.createdAt)
+ : job.createdAt;
+ let detachedSuspensionAction: AgentEventActorDetachedAction | null = null;
+ if (job.agentEventBindingId != null) {
+ const detachedActionOwner = {
+ deliveryKey: job.agentEventDeliveryKey,
+ user: job.userId,
+ ...(job.tenantId == null ? {} : { tenantId: job.tenantId }),
+ bindingId: job.agentEventBindingId,
+ conversationId,
+ generationCreatedAt: handlingGenerationCreatedAt,
+ };
+ const retainedTerminalEvidence = parseAgentEventDetachedTerminalEvidence(
+ job.agentEventDetachedTerminalEvidence,
+ );
+ if (job.agentEventDetachedTerminalEvidence != null && retainedTerminalEvidence == null) {
+ throw new Error('Detached Event Actor terminal retry evidence is invalid');
+ }
+ if (retainedTerminalEvidence != null) {
+ if (
+ retainedTerminalEvidence.deliveryKey !== job.agentEventDeliveryKey ||
+ retainedTerminalEvidence.generationCreatedAt !== handlingGenerationCreatedAt
+ ) {
+ throw new Error('Detached Event Actor terminal retry evidence is stale');
+ }
+ const replayed = await methods.settleAgentEventActorDetachedAction({
+ ...detachedActionOwner,
+ taskId: retainedTerminalEvidence.taskId,
+ idempotencyKey: retainedTerminalEvidence.idempotencyKey,
+ status: retainedTerminalEvidence.status,
+ ...(retainedTerminalEvidence.result == null
+ ? {}
+ : { result: retainedTerminalEvidence.result }),
+ ...(retainedTerminalEvidence.error == null
+ ? {}
+ : { error: retainedTerminalEvidence.error }),
+ observedAt: new Date(retainedTerminalEvidence.observedAt),
+ });
+ if (replayed.status === 'conflict') {
+ throw new Error('Detached Event Actor terminal retry evidence conflicts with action');
+ }
+ }
+ detachedSuspensionAction =
+ await methods.getAgentEventActorDetachedAction(detachedActionOwner);
+ const recoveryObservedAt = new Date();
+ if (
+ detachedSuspensionAction != null &&
+ ['reserved', 'running'].includes(detachedSuspensionAction.status) &&
+ detachedSuspensionAction.recoveryAfter <= recoveryObservedAt
+ ) {
+ await methods.markAgentEventActorDetachedActionLaunchIndeterminate({
+ ...detachedActionOwner,
+ taskId: detachedSuspensionAction.taskId,
+ idempotencyKey: detachedSuspensionAction.idempotencyKey,
+ observedAt: recoveryObservedAt,
+ });
+ /** Re-read after the CAS so a concurrent exact terminal callback wins
+ * over recovery and can immediately continue the suspended actor. */
+ detachedSuspensionAction =
+ await methods.getAgentEventActorDetachedAction(detachedActionOwner);
+ }
+ if (
+ detachedSuspensionAction?.status === 'failed' ||
+ detachedSuspensionAction?.status === 'cancelled' ||
+ detachedSuspensionAction?.status === 'launch_indeterminate'
+ ) {
+ detachedTerminalFailure =
+ detachedSuspensionAction.error ??
+ (detachedSuspensionAction.status === 'launch_indeterminate'
+ ? 'Detached expected action launch is indeterminate'
+ : `Detached expected action ${detachedSuspensionAction.status}`);
+ }
+ if (
+ job.agentEventSuspension == null &&
+ completionDeliveryKey != null &&
+ (detachedSuspensionAction?.status === 'failed' ||
+ detachedSuspensionAction?.status === 'cancelled') &&
+ unprojectedSuspension?.status === 'claimed' &&
+ unprojectedSuspension.resumeAttemptId === completionDeliveryKey &&
+ unprojectedSuspension.suspension.invocation.invocationId === job.agentEventDeliveryKey
+ ) {
+ /** A resumed hop can persist exact negative terminal evidence before
+ * replacing its claimed predecessor with the successor suspension.
+ * The completion delivery key is also the resume-attempt fence, so it
+ * authorizes retiring only this generation's claimed predecessor. */
+ retiredWithoutAction = unprojectedSuspension;
+ const cancellation = await cancelAgentEventActor(
+ {
+ ...owner,
+ suspension: unprojectedSuspension.suspension,
+ cancelAttemptId: `terminal:${job.createdAt}`,
+ reason: 'cancelled',
+ claimedResumeAttemptId: completionDeliveryKey,
+ },
+ { cancelSuspension: methods.cancelAgentEventActorSuspension },
+ );
+ if (cancellation.status !== 'cancelled') {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached predecessor cancellation is indeterminate`,
+ );
+ }
+ snapshot = await methods.getAgentEventActorSnapshot(owner);
+ }
+ /** An aborted/error generation does not prove its detached side effect
+ * stopped. Retain the original delivery and suspension until exact
+ * terminal evidence arrives or the recovery fence records uncertainty. */
+ if (
+ isIrrecoverablyTerminal &&
+ detachedSuspensionAction != null &&
+ ['reserved', 'running'].includes(detachedSuspensionAction.status)
+ ) {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached action is still in flight`,
+ );
+ }
+ if (detachedSuspensionAction?.status === 'launch_indeterminate') {
+ /** Quarantine is action truth, not a generation failure. Keep the
+ * original delivery open so a late exact callback remains admissible;
+ * no generic retry or ordinary cancellation may erase the uncertainty. */
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached action launch is indeterminate`,
+ );
+ }
+ if (isIrrecoverablyTerminal && detachedSuspensionAction?.status === 'succeeded') {
+ /** The side effect is authoritative but this aborted generation owns
+ * no committed actor checkpoint. Persist the existing reconciliation
+ * contract and wait for explicit compensation/repair instead of
+ * falsely reporting either cancellation or an applied actor turn. */
+ const marker = snapshot?.reconciliations.find(
+ (item) => item.invocationId === job.agentEventDeliveryKey,
+ );
+ if (marker?.status === 'settled' && marker.resolution === 'action_compensated') {
+ detachedTerminalFailure = 'Detached action was explicitly compensated';
+ } else {
+ if (marker?.status === 'invocation_pending') {
+ const recorded = await methods.recordAgentEventActorReconciliation({
+ ...owner,
+ reconciliation: {
+ invocationId: job.agentEventDeliveryKey,
+ actionAdmitted: marker.actionAdmitted,
+ status: 'commit_indeterminate',
+ checkpoint: marker.checkpoint,
+ action: {
+ toolName: detachedSuspensionAction.toolName,
+ toolCallId: detachedSuspensionAction.toolCallId,
+ },
+ error: 'Detached action succeeded after its generation terminated',
+ observedAt:
+ detachedSuspensionAction.settledAt ?? detachedSuspensionAction.observedAt,
+ },
+ });
+ if (!recorded) {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached reconciliation could not be recorded`,
+ );
+ }
+ } else if (marker?.status !== 'commit_indeterminate') {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached reconciliation owner is unavailable`,
+ );
+ }
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached success requires commit_indeterminate reconciliation`,
+ );
+ }
+ }
+ if (
+ unprojectedSuspension == null &&
+ (detachedSuspensionAction?.status === 'failed' ||
+ detachedSuspensionAction?.status === 'cancelled')
+ ) {
+ /** Exact negative terminal evidence proves the detached side effect no
+ * longer owns a future actor resume. Retire the pre-suspension
+ * invocation fence before releasing delivery-side admission; replay
+ * can repair either half independently. */
+ const marker = snapshot?.reconciliations.find(
+ (item) => item.invocationId === job.agentEventDeliveryKey,
+ );
+ if (marker?.status === 'invocation_pending') {
+ const abandoned = await methods.resolveAgentEventActorReconciliation({
+ ...owner,
+ invocationId: job.agentEventDeliveryKey,
+ checkpoint: marker.checkpoint,
+ expectedActionAdmitted: true,
+ resolution: 'invocation_abandoned',
+ });
+ snapshot = await methods.getAgentEventActorSnapshot(owner);
+ const remaining = snapshot?.reconciliations.find(
+ (item) => item.invocationId === job.agentEventDeliveryKey,
+ );
+ if (!abandoned && remaining?.status === 'invocation_pending') {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached terminal lifecycle could not be retired`,
+ );
+ }
+ detachedTerminalRetiredWithoutSuspension = remaining == null;
+ } else if (marker == null) {
+ /** Re-entry after the conversation-side retirement committed but
+ * before delivery admission was released. */
+ detachedTerminalRetiredWithoutSuspension = true;
+ }
+ }
+ }
+ if (
+ unprojectedSuspension?.kind === 'internal_completion' &&
+ unprojectedSuspension.status === 'pending' &&
+ unprojectedSuspension.suspension.invocation.invocationId === job.agentEventDeliveryKey &&
+ !isIrrecoverablyTerminal
+ ) {
+ const projection = job.agentEventSuspension;
+ if (
+ projection == null ||
+ projection.suspensionId !== unprojectedSuspension.suspension.suspensionId ||
+ projection.attempt !== unprojectedSuspension.suspension.attempt ||
+ job.agentEventBindingId == null
+ ) {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} internal suspension projection is stale`,
+ );
+ }
+ const action = detachedSuspensionAction;
+ if (action?.status === 'launch_indeterminate') {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached action launch is indeterminate`,
+ );
+ }
+ if (action == null || !['succeeded', 'failed', 'cancelled'].includes(action.status)) {
+ throw new Error(
+ `Agent event actor ${job.agentEventDeliveryKey} detached action is not terminal`,
+ );
+ }
+ if (options.resumeDetachedAction == null) {
+ throw new Error('Detached Event Actor resume adapter is unavailable');
+ }
+ await options.resumeDetachedAction({
+ streamId,
+ job,
+ handlingGenerationCreatedAt,
+ suspension: unprojectedSuspension.suspension,
+ action,
+ });
+ /** The successor is durable before its predecessor lane is retired. A
+ * crash between these writes leaves the successor safely blocked until
+ * idempotent terminal replay settles this completion delivery. */
+ await settleCompletionDelivery(outcome);
+ return;
+ }
if (
job.agentEventSuspension == null &&
isIrrecoverablyTerminal &&
@@ -490,17 +738,28 @@ export function createAgentEventTerminalHandler(methods: {
* admission was not yet released. Only exact closed no-action evidence is
* eligible; a committed suspension represents an applied action. */
const closed = snapshot?.suspension;
+ const closedDetachedPredecessor =
+ job.agentEventSuspension == null &&
+ completionDeliveryKey != null &&
+ closed?.status === 'closed' &&
+ (closed.outcome === 'settled' ||
+ (closed.outcome === 'cancelled' &&
+ (detachedSuspensionAction?.status === 'failed' ||
+ detachedSuspensionAction?.status === 'cancelled'))) &&
+ closed.resumeAttemptId === completionDeliveryKey &&
+ closed.suspension.invocation.invocationId === job.agentEventDeliveryKey;
if (
retiredWithoutAction == null &&
closed?.status === 'closed' &&
(closed.outcome === 'settled' || closed.outcome === 'cancelled') &&
- closed.jobCreatedAt === job.createdAt &&
closed.suspension.invocation.invocationId === job.agentEventDeliveryKey &&
- (closed.outcome === 'settled'
- ? closed.resumeAttemptId != null && closed.resumeAttemptId === job.providerExecutionId
- : isIrrecoverablyTerminal &&
- (closed.resumeAttemptId == null ||
- closed.resumeAttemptId !== job.providerExecutionStartedId))
+ (closedDetachedPredecessor ||
+ (closed.jobCreatedAt === job.createdAt &&
+ (closed.outcome === 'settled'
+ ? closed.resumeAttemptId != null && closed.resumeAttemptId === job.providerExecutionId
+ : isIrrecoverablyTerminal &&
+ (closed.resumeAttemptId == null ||
+ closed.resumeAttemptId !== job.providerExecutionStartedId))))
) {
retiredWithoutAction = closed;
}
@@ -513,8 +772,7 @@ export function createAgentEventTerminalHandler(methods: {
);
if (
retiredAdmissionId == null &&
- snapshot == null &&
- isIrrecoverablyTerminal &&
+ ((snapshot == null && isIrrecoverablyTerminal) || detachedTerminalRetiredWithoutSuspension) &&
job.agentEventBindingId != null
) {
retiredAdmissionId = await methods.getAgentEventActorActionAdmission({
@@ -570,7 +828,7 @@ export function createAgentEventTerminalHandler(methods: {
...(job.tenantId == null ? {} : { tenantId: job.tenantId }),
bindingId: durableReceipt.bindingId,
conversationId,
- generationCreatedAt: job.createdAt,
+ generationCreatedAt: handlingGenerationCreatedAt,
status: durableReceipt.resolution === 'action_compensated' ? 'failed' : 'applied',
settledAt: durableReceipt.settledAt,
...(durableReceipt.resolution === 'action_compensated' && {
@@ -626,7 +884,7 @@ export function createAgentEventTerminalHandler(methods: {
const terminalized = await methods.settleAgentTriggerHandlingOutcome({
deliveryKey: job.agentEventDeliveryKey,
conversationId,
- generationCreatedAt: job.createdAt,
+ generationCreatedAt: handlingGenerationCreatedAt,
status: legacyStatus,
settledAt: lifecycle.observedAt,
...(legacyError == null ? {} : { error: legacyError }),
@@ -643,7 +901,7 @@ export function createAgentEventTerminalHandler(methods: {
...(job.tenantId == null ? {} : { tenantId: job.tenantId }),
bindingId: job.agentEventBindingId,
conversationId,
- generationCreatedAt: job.createdAt,
+ generationCreatedAt: handlingGenerationCreatedAt,
status: legacyStatus,
settledAt: lifecycle.observedAt,
...(legacyError == null ? {} : { error: legacyError }),
@@ -704,7 +962,7 @@ export function createAgentEventTerminalHandler(methods: {
...(job.tenantId == null ? {} : { tenantId: job.tenantId }),
bindingId: job.agentEventBindingId,
conversationId,
- generationCreatedAt: job.createdAt,
+ generationCreatedAt: handlingGenerationCreatedAt,
status: 'applied',
settledAt,
...(lifecycle.actionAdmitted === true && { requiresActionAdmission: true }),
@@ -803,6 +1061,8 @@ export function createAgentEventTerminalHandler(methods: {
settlementOutcome = { status: 'failed' };
} else if (committedAction != null) {
settlementOutcome = { status: 'applied', action: committedAction };
+ } else if (detachedTerminalFailure != null) {
+ settlementOutcome = { status: 'failed' };
}
if (job.agentEventLegacyTurnToken != null && snapshot?.legacyTurn != null) {
if (snapshot.legacyTurn.token !== job.agentEventLegacyTurnToken) {
@@ -832,23 +1092,35 @@ export function createAgentEventTerminalHandler(methods: {
}
}
if (actorReceiptSettled) {
+ await settleCompletionDelivery(
+ settlementOutcome,
+ compensated
+ ? 'Applied event actor action was explicitly compensated'
+ : (detachedTerminalFailure ?? receivedJob.error ?? 'Generation failed'),
+ );
return;
}
const settled = await methods.settleAgentTriggerHandlingOutcome({
deliveryKey: job.agentEventDeliveryKey,
conversationId,
- generationCreatedAt: job.createdAt,
+ generationCreatedAt: handlingGenerationCreatedAt,
status: settlementOutcome.status,
settledAt,
...(settlementOutcome.status === 'failed' && {
error: compensated
? 'Applied event actor action was explicitly compensated'
- : (job.error ?? 'Generation failed'),
+ : (detachedTerminalFailure ?? job.error ?? 'Generation failed'),
}),
...(settlementOutcome.action != null && { action: settlementOutcome.action }),
});
if (!settled) {
throw new Error(`Failed to settle agent event delivery ${job.agentEventDeliveryKey}`);
}
+ await settleCompletionDelivery(
+ settlementOutcome,
+ compensated
+ ? 'Applied event actor action was explicitly compensated'
+ : (detachedTerminalFailure ?? receivedJob.error ?? 'Generation failed'),
+ );
};
}
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index 22e5bf57a2..de458161e3 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -1,4 +1,8 @@
-import { getTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas';
+import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ getTenantId,
+ SYSTEM_TENANT_ID,
+} from '@librechat/data-schemas';
import type { AgentTriggerDeliveryPersistence, AgentTriggerStoredRecord } from './service';
import { AgentTriggerServiceUnavailableError, createAgentTriggerService } from './service';
import { __resetShutdownStateForTests } from '../../app/shutdown';
@@ -157,6 +161,7 @@ describe('durable agent trigger service', () => {
methods,
mintToken: () => 'token',
fetch: async () => new Response('{}', { status: 500 }),
+ supportsDetachedActionCompletion: () => true,
deliveryOptions: { concurrency: 1, tickMs: 60_000 },
});
@@ -168,6 +173,11 @@ describe('durable agent trigger service', () => {
expect(methods.ensureAgentTriggerDeliveryIndexes).toHaveBeenCalledTimes(1);
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
+ expect.objectContaining({
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],
+ }),
+ );
expect(methods.enqueueAgentTriggerDelivery).toHaveBeenCalledWith(
expect.objectContaining({
deliveryKey: expect.stringMatching(/^trigger_/),
@@ -187,6 +197,22 @@ describe('durable agent trigger service', () => {
await service.stop();
});
+ it('does not advertise detached completion capability without durable generation storage', async () => {
+ const methods = deliveryMethods();
+ const service = createAgentTriggerService({
+ methods,
+ supportsDetachedActionCompletion: () => false,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+
+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
+
+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
+ expect.objectContaining({ workerCapabilities: [] }),
+ );
+ await service.stop();
+ });
+
it('persists terminal handling serialization only for bound continuations', async () => {
const methods = deliveryMethods();
const service = createAgentTriggerService({
diff --git a/packages/api/src/agents/triggers/service.spec.ts b/packages/api/src/agents/triggers/service.spec.ts
index 51fe993731..846004b557 100644
--- a/packages/api/src/agents/triggers/service.spec.ts
+++ b/packages/api/src/agents/triggers/service.spec.ts
@@ -1,5 +1,9 @@
import jwt from 'jsonwebtoken';
import type { AgentTriggerFetch } from './host';
+import {
+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,
+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+} from './detachedAction';
import { createAgentTriggerEnvelope } from './envelope';
import { AGENT_TRIGGER_SCOPE } from '../../crypto/jwt';
import { createAgentTriggerService } from './service';
@@ -22,6 +26,38 @@ const envelope = () =>
input: 'Handle the ready resource.',
});
+const detachedCompletionEnvelope = () =>
+ createAgentTriggerEnvelope({
+ mode: 'continue',
+ requestId: 'request-detached-1',
+ deliveryId: 'delivery-detached-1',
+ receivedAt: 20,
+ principal: { id: 'user-1', role: 'member', tenantId: 'tenant-1' },
+ target: {
+ agentId: 'agent-1',
+ conversationId: 'conversation-1',
+ parentMessageId: 'parent-1',
+ bindingId: 'binding-1',
+ sourceKeyId: 'source-key-1',
+ },
+ event: {
+ id: 'event_actor_task_1',
+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,
+ occurredAt: 10,
+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },
+ payload: {
+ version: 1,
+ invocationId: 'trigger-original-1',
+ generationCreatedAt: 1,
+ wakeGenerationCreatedAt: 2,
+ taskId: 'event_actor_task_1',
+ idempotencyKey: 'a'.repeat(64),
+ },
+ },
+ input: 'Resume the actor.',
+ expectedAction: { toolName: 'submit_move' },
+ });
+
const accepted = () =>
new Response(
JSON.stringify({
@@ -91,6 +127,19 @@ describe('createAgentTriggerService', () => {
);
});
+ it('keeps detached completions on the capable local replica despite a self-URL override', async () => {
+ process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal/base';
+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
+ async () => accepted(),
+ );
+ const service = createAgentTriggerService({ fetch: fetcher, mintToken: () => 'token' });
+ await service.initialize({ address: { address: '0.0.0.0', family: 'IPv4', port: 4123 } });
+
+ await service.dispatch(detachedCompletionEnvelope());
+
+ expect(String(fetcher.mock.calls[0][0])).toBe('http://127.0.0.1:4123/api/agents/chat/agents');
+ });
+
it('rechecks the durable principal before dispatching queued or direct work', async () => {
process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal';
const fetcher = jest.fn(async () => accepted());
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index 549630c0fe..554471c291 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -1,4 +1,8 @@
-import { logger, runAsSystem } from '@librechat/data-schemas';
+import {
+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ logger,
+ runAsSystem,
+} from '@librechat/data-schemas';
import type { AgentTriggerDeliveryStatusRecord } from '@librechat/data-schemas';
import type {
AgentTriggerDeliveryFailure,
@@ -45,6 +49,7 @@ export interface AgentTriggerServiceDeps {
userDrainPollMs?: number;
purgeRecoveryIntervalMs?: number;
purgeRecoveryLimit?: number;
+ supportsDetachedActionCompletion?: () => boolean;
}
export interface AgentTriggerDeliveryReceipt {
@@ -86,7 +91,11 @@ export interface AgentTriggerDeliveryPersistence {
enqueueAgentTriggerDelivery: (
input: PreparedAgentTriggerDelivery,
) => Promise<{ delivery: AgentTriggerStoredRecord; replayed: boolean }>;
- claimNextAgentTriggerDelivery: AgentTriggerDeliveryStore['claimNext'];
+ claimNextAgentTriggerDelivery: (
+ input: Parameters<AgentTriggerDeliveryStore['claimNext']>[0] & {
+ workerCapabilities?: string[];
+ },
+ ) => ReturnType<AgentTriggerDeliveryStore['claimNext']>;
findEarlierAgentTriggerDelivery: AgentTriggerDeliveryStore['findEarlierUnsettled'];
getAgentTriggerDeliveryBatch: AgentTriggerDeliveryStore['getBatch'];
releaseAgentTriggerDelivery: AgentTriggerDeliveryStore['release'];
@@ -148,9 +157,18 @@ export interface AgentTriggerService {
purgeUser: (userId: string) => Promise<void>;
}
-function createDeliveryStore(methods: AgentTriggerDeliveryPersistence): AgentTriggerDeliveryStore {
+function createDeliveryStore(
+ methods: AgentTriggerDeliveryPersistence,
+ supportsDetachedActionCompletion: () => boolean,
+): AgentTriggerDeliveryStore {
return {
- claimNext: methods.claimNextAgentTriggerDelivery,
+ claimNext: (input) =>
+ methods.claimNextAgentTriggerDelivery({
+ ...input,
+ workerCapabilities: supportsDetachedActionCompletion()
+ ? [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1]
+ : [],
+ }),
findEarlierUnsettled: methods.findEarlierAgentTriggerDelivery,
getBatch: methods.getAgentTriggerDeliveryBatch,
release: methods.releaseAgentTriggerDelivery,
@@ -165,7 +183,16 @@ function createDeliveryStore(methods: AgentTriggerDeliveryPersistence): AgentTri
function publicReceiptStatus(
status: AgentTriggerStoredRecord['status'],
): AgentTriggerDeliveryReceipt['status'] {
- return status === 'batched' ? 'pending' : status;
+ if (status === 'batched' || status === 'capability_pending') {
+ return 'pending';
+ }
+ if (status === 'capability_staging') {
+ return 'staging';
+ }
+ if (status === 'capability_dead') {
+ return 'dead';
+ }
+ return status === 'capability_leased' ? 'leased' : status;
}
function requireDeliveryOrigin(boundOrigin: string | undefined): void {
@@ -192,6 +219,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
const purgeRecoveryIntervalMs =
deps.purgeRecoveryIntervalMs ?? DEFAULT_PURGE_RECOVERY_INTERVAL_MS;
const purgeRecoveryLimit = deps.purgeRecoveryLimit ?? DEFAULT_PURGE_RECOVERY_LIMIT;
+ const supportsDetachedActionCompletion = deps.supportsDetachedActionCompletion ?? (() => false);
if (!Number.isSafeInteger(userDrainTimeoutMs) || userDrainTimeoutMs <= 0) {
throw new TypeError('userDrainTimeoutMs must be a positive integer');
}
@@ -213,8 +241,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
let stopping = false;
const isPrincipalActive = deps.isPrincipalActive;
const host: AgentTriggerExecutionHost = createAgentTriggerExecutionHost({
- getBaseUrl: () => {
- const origin = process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin;
+ getBaseUrl: (options) => {
+ const origin =
+ options?.localOnly === true
+ ? boundOrigin
+ : (process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin);
if (origin == null) {
throw new Error('Agent trigger service has not been initialized with a listener address');
}
@@ -391,7 +422,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
deliveryEngine = createAgentTriggerDeliveryEngine(
{
- store: createDeliveryStore(methods),
+ store: createDeliveryStore(methods, supportsDetachedActionCompletion),
dispatch: dispatchForActivePrincipal,
},
deps.deliveryOptions,
diff --git a/packages/api/src/agents/triggers/types.ts b/packages/api/src/agents/triggers/types.ts
index ea44518061..40963438de 100644
--- a/packages/api/src/agents/triggers/types.ts
+++ b/packages/api/src/agents/triggers/types.ts
@@ -22,3 +22,17 @@ export interface AgentEventAppliedAction {
toolName: string;
toolCallId?: string;
}
+
+/** Job-store outbox record for terminal detached-action evidence that has not
+ * necessarily reached the authoritative delivery row yet. */
+export interface AgentEventDetachedTerminalEvidence {
+ version: 1;
+ deliveryKey: string;
+ generationCreatedAt: number;
+ taskId: string;
+ idempotencyKey: string;
+ status: 'succeeded' | 'failed' | 'cancelled';
+ result?: string;
+ error?: string;
+ observedAt: number;
+}
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 9c563497fb..fb35be40e9 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -915,6 +915,26 @@ class GenerationJobManagerClass {
this.terminalHostActionHandler = handler;
}
+ /**
+ * Event-driven retry for a durable terminal host action whose external
+ * evidence arrived after the generation itself became terminal. The exact
+ * generation fence prevents a delayed callback from touching a replacement;
+ * the periodic recovery sweep remains the restart/replica fallback.
+ */
+ async retryTerminalHostAction(streamId: string, expectedCreatedAt: number): Promise<boolean> {
+ const job = await this.jobStore.getJob(streamId);
+ if (
+ job?.createdAt !== expectedCreatedAt ||
+ job.status === 'running' ||
+ job.status === 'requires_action' ||
+ job.providerDrained === false ||
+ job.terminalHostActionPending !== true
+ ) {
+ return false;
+ }
+ return this.runTerminalHostActionHandler(streamId, job);
+ }
+
private get storeLabel(): GenerationJobStore {
return this._isRedis ? 'redis' : 'memory';
}
@@ -2606,6 +2626,10 @@ class GenerationJobManagerClass {
// Surface whether the turn was temporary so a resume keeps it non-persisted.
isTemporary: jobData.isTemporary,
agentEventDeliveryKey: jobData.agentEventDeliveryKey,
+ agentEventInvocationKey: jobData.agentEventInvocationKey,
+ agentEventInvocationGenerationCreatedAt: jobData.agentEventInvocationGenerationCreatedAt,
+ agentEventDetachedActionProducerRequired: jobData.agentEventDetachedActionProducerRequired,
+ agentEventDetachedTerminalEvidence: jobData.agentEventDetachedTerminalEvidence,
agentEventBindingId: jobData.agentEventBindingId,
agentEventExpectedAction: jobData.agentEventExpectedAction,
agentEventSuspension: jobData.agentEventSuspension,
@@ -6699,6 +6723,34 @@ class GenerationJobManagerClass {
await this.jobStore.updateJob(streamId, updates, generationId);
}
+ /** Stages exact detached terminal evidence in the generation-owned job outbox
+ * and verifies the epoch-fenced write before the external result leaves memory. */
+ async persistAgentEventDetachedTerminalEvidence(
+ streamId: string,
+ expectedCreatedAt: number,
+ evidence: NonNullable<t.GenerationJobMetadata['agentEventDetachedTerminalEvidence']>,
+ ): Promise<boolean> {
+ await this.jobStore.updateJob(
+ streamId,
+ { agentEventDetachedTerminalEvidence: evidence },
+ expectedCreatedAt,
+ );
+ const persisted = await this.jobStore.getJob(streamId);
+ const actual = persisted?.agentEventDetachedTerminalEvidence;
+ return (
+ persisted?.createdAt === expectedCreatedAt &&
+ actual?.version === evidence.version &&
+ actual.deliveryKey === evidence.deliveryKey &&
+ actual.generationCreatedAt === evidence.generationCreatedAt &&
+ actual.taskId === evidence.taskId &&
+ actual.idempotencyKey === evidence.idempotencyKey &&
+ actual.status === evidence.status &&
+ actual.result === evidence.result &&
+ actual.error === evidence.error &&
+ actual.observedAt === evidence.observedAt
+ );
+ }
+
/** Records that one exact provider segment has completed every trailing write.
* The opaque segment id prevents a paused controller from acknowledging a
* later HITL resume that reuses the same generation epoch. */
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
index 3ec4efa182..bc70dba462 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
@@ -339,6 +339,10 @@ describe('RedisJobStore', () => {
model: 'test-model',
agent_id: 'agent-1',
isTemporary: false,
+ agentEventDeliveryKey: 'completion-delivery-1',
+ agentEventInvocationKey: 'original-delivery-1',
+ agentEventInvocationGenerationCreatedAt: 987654,
+ agentEventDetachedActionProducerRequired: true,
scheduleId: 'schedule-1',
scheduledFor: '2026-08-17T12:00:00.000Z',
scheduleConfigRevision: 4,
@@ -397,6 +401,10 @@ describe('RedisJobStore', () => {
parentMessageId: 'parent-1',
},
responseMessageId: 'response-1',
+ agentEventDeliveryKey: 'completion-delivery-1',
+ agentEventInvocationKey: 'original-delivery-1',
+ agentEventInvocationGenerationCreatedAt: 987654,
+ agentEventDetachedActionProducerRequired: true,
mcpRequestBody: {
messageId: 'response-1',
conversationId: 'overridden-conversation',
@@ -495,6 +503,35 @@ describe('RedisJobStore', () => {
});
});
+ test('restores detached terminal outbox evidence from Redis', async () => {
+ const evidence = {
+ version: 1 as const,
+ deliveryKey: 'trigger-1',
+ generationCreatedAt: 100,
+ taskId: 'task-1',
+ idempotencyKey: 'a'.repeat(64),
+ status: 'succeeded' as const,
+ result: 'accepted',
+ observedAt: 150,
+ };
+ const redis = {
+ isCluster: true,
+ hgetall: jest.fn().mockResolvedValue({
+ streamId: 'stream-detached-terminal',
+ userId: 'user-1',
+ status: 'complete',
+ createdAt: '100',
+ syncSent: '0',
+ agentEventDetachedTerminalEvidence: JSON.stringify(evidence),
+ }),
+ } as unknown as Cluster;
+ const store = new RedisJobStore(redis);
+
+ await expect(store.getJob('stream-detached-terminal')).resolves.toMatchObject({
+ agentEventDetachedTerminalEvidence: evidence,
+ });
+ });
+
test('atomically resets predecessor state when creating a replacement', async () => {
const evalJobCreation = jest
.fn()
diff --git a/packages/api/src/stream/__tests__/terminalHostAction.spec.ts b/packages/api/src/stream/__tests__/terminalHostAction.spec.ts
index 2bf81b5953..a30785acdc 100644
--- a/packages/api/src/stream/__tests__/terminalHostAction.spec.ts
+++ b/packages/api/src/stream/__tests__/terminalHostAction.spec.ts
@@ -22,6 +22,37 @@ describe('GenerationJobManager terminal host actions', () => {
await manager.destroy();
});
+ it('verifies detached terminal outbox persistence against the exact generation', async () => {
+ const streamId = 'conversation-detached-outbox';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const evidence = {
+ version: 1 as const,
+ deliveryKey: 'trigger-outbox',
+ generationCreatedAt: job.createdAt,
+ taskId: 'task-outbox',
+ idempotencyKey: 'a'.repeat(64),
+ status: 'succeeded' as const,
+ result: 'accepted',
+ observedAt: job.createdAt + 1,
+ };
+
+ await expect(
+ manager.persistAgentEventDetachedTerminalEvidence(streamId, job.createdAt, evidence),
+ ).resolves.toBe(true);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ agentEventDetachedTerminalEvidence: evidence,
+ });
+
+ const replacement = await store.createJob(streamId, 'user-1', streamId);
+ expect(replacement.createdAt).not.toBe(job.createdAt);
+ await expect(
+ manager.persistAgentEventDetachedTerminalEvidence(streamId, job.createdAt, evidence),
+ ).resolves.toBe(false);
+ await expect(store.getJob(streamId)).resolves.not.toHaveProperty(
+ 'agentEventDetachedTerminalEvidence',
+ );
+ });
+
it('runs and acknowledges a generation-fenced host action before terminal cleanup', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
@@ -89,6 +120,29 @@ describe('GenerationJobManager terminal host actions', () => {
);
});
+ it('retries an exact pending terminal host action when late evidence arrives', async () => {
+ const handler = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('detached action is still running'))
+ .mockResolvedValue(undefined);
+ manager.setTerminalHostActionHandler(handler);
+ const streamId = 'conversation-late-terminal-evidence';
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { agentEventDeliveryKey: 'trigger-late-evidence' },
+ });
+
+ await expect(manager.completeJob(streamId, undefined, job.createdAt)).resolves.toBe(true);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ terminalHostActionPending: true,
+ });
+
+ await expect(manager.retryTerminalHostAction(streamId, job.createdAt + 1)).resolves.toBe(false);
+ expect(handler).toHaveBeenCalledTimes(1);
+ await expect(manager.retryTerminalHostAction(streamId, job.createdAt)).resolves.toBe(true);
+ expect(handler).toHaveBeenCalledTimes(2);
+ await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');
+ });
+
it('waits for the provider evidence fence and retains a failed tool close', async () => {
const handler = jest.fn().mockResolvedValue(undefined);
manager.setTerminalHostActionHandler(handler);
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index 787a8c8831..a987a78350 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -4738,6 +4738,17 @@ export class RedisJobStore implements IJobStoreV2 {
agent_id: data.agent_id || undefined,
isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,
agentEventDeliveryKey: data.agentEventDeliveryKey || undefined,
+ agentEventInvocationKey: data.agentEventInvocationKey || undefined,
+ agentEventInvocationGenerationCreatedAt: data.agentEventInvocationGenerationCreatedAt
+ ? parseInt(data.agentEventInvocationGenerationCreatedAt, 10)
+ : undefined,
+ agentEventDetachedActionProducerRequired:
+ data.agentEventDetachedActionProducerRequired != null
+ ? data.agentEventDetachedActionProducerRequired === '1'
+ : undefined,
+ agentEventDetachedTerminalEvidence: data.agentEventDetachedTerminalEvidence
+ ? JSON.parse(data.agentEventDetachedTerminalEvidence)
+ : undefined,
agentEventBindingId: data.agentEventBindingId || undefined,
agentEventExpectedAction: data.agentEventExpectedAction
? JSON.parse(data.agentEventExpectedAction)
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index fde92ad465..c545bf162e 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -5,6 +5,7 @@ import type {
UserSubmittedMessageFieldPath,
} from 'librechat-data-provider';
import type { RunStep, StandardGraph } from '@librechat/agents';
+import type { AgentEventDetachedTerminalEvidence } from '~/agents/triggers/types';
import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';
import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';
import type { RecoveredSteerPayload } from '../SteerRecovery';
@@ -283,6 +284,14 @@ export interface SerializableJobData {
*/
isTemporary?: boolean;
agentEventDeliveryKey?: string;
+ /** Original actor invocation when an internal completion delivery owns this generation. */
+ agentEventInvocationKey?: string;
+ /** Original actor invocation generation retained across completion HITL resumes. */
+ agentEventInvocationGenerationCreatedAt?: number;
+ /** This generation must resume on a durable detached-action producer. */
+ agentEventDetachedActionProducerRequired?: boolean;
+ /** Durable retry payload captured before detached terminal evidence is written to Mongo. */
+ agentEventDetachedTerminalEvidence?: AgentEventDetachedTerminalEvidence;
/** Trusted actor binding copied from the authenticated delivery envelope. */
agentEventBindingId?: string;
agentEventExpectedAction?: import('~/agents/triggers/types').AgentTriggerExpectedAction;
@@ -408,6 +417,10 @@ export type JobMetadataPatch = Partial<
| 'agent_id'
| 'isTemporary'
| 'agentEventDeliveryKey'
+ | 'agentEventInvocationKey'
+ | 'agentEventInvocationGenerationCreatedAt'
+ | 'agentEventDetachedActionProducerRequired'
+ | 'agentEventDetachedTerminalEvidence'
| 'agentEventBindingId'
| 'agentEventExpectedAction'
| 'agentEventSuspension'
diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts
index 997d29b7ef..ad2f87c41b 100644
--- a/packages/api/src/stream/metadata.ts
+++ b/packages/api/src/stream/metadata.ts
@@ -39,6 +39,20 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): J
if (metadata.agentEventDeliveryKey) {
patch.agentEventDeliveryKey = metadata.agentEventDeliveryKey;
}
+ if (metadata.agentEventInvocationKey) {
+ patch.agentEventInvocationKey = metadata.agentEventInvocationKey;
+ }
+ if (metadata.agentEventInvocationGenerationCreatedAt !== undefined) {
+ patch.agentEventInvocationGenerationCreatedAt =
+ metadata.agentEventInvocationGenerationCreatedAt;
+ }
+ if (metadata.agentEventDetachedActionProducerRequired !== undefined) {
+ patch.agentEventDetachedActionProducerRequired =
+ metadata.agentEventDetachedActionProducerRequired;
+ }
+ if (metadata.agentEventDetachedTerminalEvidence) {
+ patch.agentEventDetachedTerminalEvidence = metadata.agentEventDetachedTerminalEvidence;
+ }
if (metadata.agentEventBindingId) {
patch.agentEventBindingId = metadata.agentEventBindingId;
}
diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts
index b549aa4824..5f9707951a 100644
--- a/packages/api/src/types/stream.ts
+++ b/packages/api/src/types/stream.ts
@@ -1,6 +1,7 @@
import type { Agents, UserSubmittedMessageFieldPath } from 'librechat-data-provider';
import type { EventEmitter } from 'events';
import type {
+ AgentEventDetachedTerminalEvidence,
AgentEventSuspensionProjection,
AgentTriggerExpectedAction,
} from '../agents/triggers/types';
@@ -47,6 +48,14 @@ export interface GenerationJobMetadata {
isTemporary?: boolean;
/** Exact durable delivery whose accepted continuation created this generation. */
agentEventDeliveryKey?: string;
+ /** Original actor invocation when the current mailbox delivery is an internal completion. */
+ agentEventInvocationKey?: string;
+ /** Original actor invocation generation retained across completion HITL resumes. */
+ agentEventInvocationGenerationCreatedAt?: number;
+ /** This generation must resume on a durable detached-action producer. */
+ agentEventDetachedActionProducerRequired?: boolean;
+ /** Durable retry payload captured before detached terminal evidence reaches Mongo. */
+ agentEventDetachedTerminalEvidence?: AgentEventDetachedTerminalEvidence;
/** Trusted actor binding copied from the authenticated delivery envelope. */
agentEventBindingId?: string;
/** Optional action evidence contract declared by the authenticated event source. */
diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts
index edcd562925..a887614093 100644
--- a/packages/data-schemas/src/index.ts
+++ b/packages/data-schemas/src/index.ts
@@ -53,6 +53,7 @@ export {
MAX_AGENT_EVENT_ACTOR_SUMMARY_LENGTH,
MAX_AGENT_EVENT_ACTOR_TOOL_NAME_LENGTH,
} from './types/convo';
+export { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from './types/triggerDelivery';
export type * from './types';
export type * from './methods';
export {
diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts
index 4a4a2e1fb2..815b5c8dec 100644
--- a/packages/data-schemas/src/methods/conversation.spec.ts
+++ b/packages/data-schemas/src/methods/conversation.spec.ts
@@ -4027,6 +4027,9 @@ describe('Conversation Operations', () => {
jobCreatedAt: 123,
}),
).resolves.toEqual({ status: 'stored' });
+ await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({
+ suspension: { kind: 'human_decision' },
+ });
await expect(
methods.storeAgentEventActorSuspension({
...owner,
@@ -4056,6 +4059,7 @@ describe('Conversation Operations', () => {
methods.storeAgentEventActorSuspension({
...owner,
suspension: second,
+ kind: 'internal_completion',
actionId: 'action-second',
jobCreatedAt: 123,
previous: {
@@ -4065,6 +4069,12 @@ describe('Conversation Operations', () => {
},
}),
).resolves.toEqual({ status: 'stored' });
+ await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({
+ suspension: {
+ kind: 'internal_completion',
+ suspension: { suspensionId: second.suspensionId },
+ },
+ });
await expect(
methods.claimAgentEventActorSuspension({
...owner,
diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts
index de9fb429af..a45d28cddd 100644
--- a/packages/data-schemas/src/methods/conversation.ts
+++ b/packages/data-schemas/src/methods/conversation.ts
@@ -321,6 +321,9 @@ export interface ConversationMethods {
conversationId: string;
tenantId?: string;
suspension: IAgentEventActorSuspensionEvidence;
+ kind?: 'human_decision' | 'internal_completion';
+ appliedAction?: { toolName: string; toolCallId?: string };
+ handlingGenerationCreatedAt?: number;
actionId: string;
jobCreatedAt: number;
/** The segment applied its expected action before publishing this successor pause. */
@@ -642,6 +645,9 @@ export function createConversationMethods(
conversationId: string;
tenantId?: string;
suspension: IAgentEventActorSuspensionEvidence;
+ kind?: 'human_decision' | 'internal_completion';
+ appliedAction?: { toolName: string; toolCallId?: string };
+ handlingGenerationCreatedAt?: number;
actionId: string;
jobCreatedAt: number;
invalidateHead?: boolean;
@@ -684,15 +690,19 @@ export function createConversationMethods(
...activeExpirationFilter<IConversation>(),
...predecessor,
};
+ const suspension = {
+ suspension: input.suspension,
+ kind: input.kind ?? 'human_decision',
+ ...(input.appliedAction == null ? {} : { appliedAction: input.appliedAction }),
+ handlingGenerationCreatedAt: input.handlingGenerationCreatedAt ?? input.jobCreatedAt,
+ actionId: input.actionId,
+ jobCreatedAt: input.jobCreatedAt,
+ status: 'pending' as const,
+ observedAt: new Date(),
+ };
const storeUpdate = {
$set: {
- agentEventActorSuspension: {
- suspension: input.suspension,
- actionId: input.actionId,
- jobCreatedAt: input.jobCreatedAt,
- status: 'pending' as const,
- observedAt: new Date(),
- },
+ agentEventActorSuspension: suspension,
...(input.invalidateHead === true ? { 'agentEventActor.requiresColdStart': true } : {}),
},
};
@@ -713,13 +723,7 @@ export function createConversationMethods(
{ ...ownership, agentEventActor: { $exists: false } },
{
$set: {
- agentEventActorSuspension: {
- suspension: input.suspension,
- actionId: input.actionId,
- jobCreatedAt: input.jobCreatedAt,
- status: 'pending',
- observedAt: new Date(),
- },
+ agentEventActorSuspension: suspension,
},
},
{ new: false, timestamps: false },
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index 98fa95da7a..bbb3b5c5ea 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -15,6 +15,7 @@ import {
setAgentEventActorReceiptMetricObserver,
type AgentTriggerDeliveryMethods,
} from './triggerDelivery';
+import { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from '~/types/triggerDelivery';
import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';
import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
@@ -237,6 +238,142 @@ describe('agent trigger delivery methods', () => {
expect(claims.filter((claim) => claim != null)).toHaveLength(1);
});
+ it('keeps capability-fenced work limited to capable workers through lease recovery', async () => {
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ }),
+ );
+ expect(queued.delivery).toMatchObject({
+ status: 'capability_pending',
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ });
+ const claimInput = {
+ workerId: 'capable-worker',
+ claimToken: 'capable-claim',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ };
+
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ ...claimInput,
+ workerId: 'old-worker',
+ claimToken: 'old-claim',
+ }),
+ ).resolves.toBeNull();
+ const capable = await methods.claimNextAgentTriggerDelivery({
+ ...claimInput,
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],
+ });
+ expect(capable).toMatchObject({ status: 'capability_leased' });
+
+ await Delivery.updateOne(
+ { _id: capable!.id },
+ { $set: { leaseUntil: new Date(START.getTime() - 60_000) } },
+ );
+ const recoveryNow = new Date(START.getTime() + 60_000);
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'old-worker',
+ claimToken: 'old-recovery',
+ now: recoveryNow,
+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),
+ }),
+ ).resolves.toBeNull();
+ const recovered = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'capable-worker-2',
+ claimToken: 'capable-recovery',
+ now: recoveryNow,
+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],
+ });
+ expect(recovered).toMatchObject({ status: 'capability_leased' });
+
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: recovered!.id,
+ workerId: 'capable-worker-2',
+ claimToken: 'capable-recovery',
+ now: recoveryNow,
+ });
+ expect(attempt).toBe(1);
+ await expect(
+ methods.deadLetterAgentTriggerDelivery({
+ id: recovered!.id,
+ workerId: 'capable-worker-2',
+ claimToken: 'capable-recovery',
+ attempt: 1,
+ error: transientFailure({ attemptedAt: recoveryNow }),
+ settledAt: recoveryNow,
+ }),
+ ).resolves.toBe(true);
+ await expect(Delivery.findById(recovered!.id).lean()).resolves.toMatchObject({
+ status: 'capability_dead',
+ });
+
+ const [deadLetter] = await methods.getAgentTriggerDeadLetters();
+ expect(deadLetter).toMatchObject({ id: recovered!.id, status: 'capability_dead' });
+ const requeued = await methods.requeueAgentTriggerDelivery(recovered!.id, recoveryNow);
+ expect(requeued).toMatchObject({ status: 'capability_pending' });
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'old-worker',
+ claimToken: 'old-requeue',
+ now: recoveryNow,
+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),
+ }),
+ ).resolves.toBeNull();
+ const finalClaim = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'capable-worker-3',
+ claimToken: 'capable-final',
+ now: recoveryNow,
+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],
+ });
+ const finalAttempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: finalClaim!.id,
+ workerId: 'capable-worker-3',
+ claimToken: 'capable-final',
+ now: recoveryNow,
+ });
+ await expect(
+ methods.completeAgentTriggerDelivery({
+ id: finalClaim!.id,
+ workerId: 'capable-worker-3',
+ claimToken: 'capable-final',
+ attempt: finalAttempt!,
+ result: { status: 'complete' },
+ settledAt: recoveryNow,
+ }),
+ ).resolves.toBe(true);
+ });
+
+ it('claims the oldest eligible delivery without capability traffic priority', async () => {
+ const ordinary = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'ordinary-older',
+ availableAt: new Date(START.getTime() - 2_000),
+ }),
+ );
+ await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ deliveryKey: 'capability-newer',
+ availableAt: new Date(START.getTime() - 1_000),
+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
+ }),
+ );
+
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'capable-worker',
+ claimToken: 'claim-oldest',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],
+ }),
+ ).resolves.toMatchObject({ id: ordinary.delivery.id, status: 'leased' });
+ });
+
it('allocates a replica-stable monotonic sequence within an ordering lane', async () => {
const user = new mongoose.Types.ObjectId();
const replicas = [
@@ -256,6 +393,86 @@ describe('agent trigger delivery methods', () => {
);
});
+ it('does not publish a later requeue through a stale lane reservation', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'stale-publication-generation';
+ const originalUpdateOne = Delivery.collection.updateOne.bind(Delivery.collection);
+ let releaseStalePublication!: () => void;
+ let stalePublicationPaused!: () => void;
+ const stalePublicationGate = new Promise<void>((resolve) => {
+ releaseStalePublication = resolve;
+ });
+ const stalePublicationObserved = new Promise<void>((resolve) => {
+ stalePublicationPaused = resolve;
+ });
+ let paused = false;
+ const updateSpy = jest
+ .spyOn(Delivery.collection, 'updateOne')
+ .mockImplementation(async (filter, update, options) => {
+ const status = filter.status as { $in?: string[] } | undefined;
+ if (
+ !paused &&
+ filter.orderingKey === orderingKey &&
+ status?.$in?.includes('staging') === true &&
+ filter.requeueCount === 0
+ ) {
+ paused = true;
+ stalePublicationPaused();
+ await stalePublicationGate;
+ }
+ return originalUpdateOne(filter, update, options);
+ });
+
+ try {
+ const enqueue = methods.enqueueAgentTriggerDelivery(enqueueInput({ user, orderingKey }));
+ await stalePublicationObserved;
+
+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(1);
+ const published = await Delivery.findOne({ orderingKey }).lean();
+ expect(published).toMatchObject({ status: 'pending', laneSequence: 1, requeueCount: 0 });
+
+ await originalUpdateOne(
+ { _id: published!._id },
+ {
+ $set: {
+ status: 'staging',
+ laneSequence: 0,
+ requeueCount: 1,
+ stagingRecoveryAt: new Date(),
+ },
+ },
+ );
+ await LaneSequence.updateOne(
+ {
+ _id: orderingKey,
+ value: 1,
+ publisherDeliveryId: { $exists: false },
+ },
+ {
+ $inc: { value: 1 },
+ $set: {
+ publisherDeliveryId: published!._id,
+ publisherRequeueCount: 1,
+ publisherStartedAt: new Date(),
+ },
+ },
+ );
+
+ releaseStalePublication();
+ await expect(enqueue).resolves.toMatchObject({
+ delivery: { status: 'pending', laneSequence: 2, requeueCount: 1 },
+ });
+ await expect(Delivery.findById(published!._id).lean()).resolves.toMatchObject({
+ status: 'pending',
+ laneSequence: 2,
+ requeueCount: 1,
+ });
+ } finally {
+ releaseStalePublication();
+ updateSpy.mockRestore();
+ }
+ });
+
it('coalesces a concurrent replica burst behind one claimable root and settles every receipt', async () => {
const user = new mongoose.Types.ObjectId();
const coalesceUntil = new Date(Date.now() + 60_000);
@@ -1758,6 +1975,14 @@ describe('agent trigger delivery methods', () => {
await expect(methods.admitAgentEventActorAction(actionAdmission)).resolves.toBe(true);
await expect(methods.settleAgentEventActorReceipt(settlement)).resolves.toBe(true);
await expect(methods.releaseAgentEventActorAction(actionAdmission)).resolves.toBe(false);
+ const settledDelivery = await Delivery.findOne({ deliveryKey: settlement.deliveryKey })
+ .select('+actorActionAdmittedAt +actorActionAdmissionId')
+ .lean();
+ expect(settledDelivery).not.toHaveProperty('actorActionAdmittedAt');
+ expect(settledDelivery).not.toHaveProperty('actorActionAdmissionId');
+ await expect(
+ methods.countActiveAgentTriggerDeliveriesByUser(queued.delivery.user, settlement.settledAt),
+ ).resolves.toBe(0);
/** Simulate a receipt written by the pre-normalization build after
* transport dead-lettered the root. Exact replay must retire that dead
* state instead of leaving a non-requeueable dead letter. */
@@ -1833,6 +2058,474 @@ describe('agent trigger delivery methods', () => {
).resolves.toBeNull();
});
+ it('persists detached evidence before the started transport response settles', async () => {
+ const bindingId = 'binding-detached-leased';
+ const conversationId = 'conversation-detached-leased';
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ awaitTerminalHandling: true,
+ tenantId: undefined,
+ envelope: {
+ mode: 'continue',
+ target: { bindingId },
+ event: { source: { id: 'source-key-1', type: 'remote_api_key' } },
+ },
+ }),
+ );
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-detached-leased',
+ claimToken: 'claim-detached-leased',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: claimed!.id,
+ workerId: 'worker-detached-leased',
+ claimToken: 'claim-detached-leased',
+ now: START,
+ });
+ const generationCreatedAt = START.getTime() + 1_000;
+ await expect(
+ methods.admitAgentEventActorAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ admittedAt: new Date(generationCreatedAt),
+ admissionId: 'admission-detached-leased',
+ }),
+ ).resolves.toBe(true);
+ const reservation = await methods.reserveAgentEventActorDetachedAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ turnId: 'response-detached-leased:0',
+ invocationId: queued.delivery.deliveryKey,
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-detached-leased',
+ reservedAt: new Date(generationCreatedAt + 1_000),
+ recoveryAfter: new Date(generationCreatedAt + 61_000),
+ });
+ expect(reservation.status).toBe('reserved');
+ const update = {
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ taskId: reservation.action.taskId,
+ idempotencyKey: reservation.action.idempotencyKey,
+ observedAt: new Date(generationCreatedAt + 2_000),
+ recoveryAfter: new Date(generationCreatedAt + 1_802_000),
+ };
+ await expect(methods.markAgentEventActorDetachedActionRunning(update)).resolves.toEqual({
+ status: 'applied',
+ });
+ const retryAt = new Date(generationCreatedAt + 3_000);
+ await expect(
+ methods.retryAgentTriggerDelivery({
+ id: claimed!.id,
+ workerId: 'worker-detached-leased',
+ claimToken: 'claim-detached-leased',
+ attempt: attempt!,
+ error: transientFailure({ attemptedAt: retryAt }),
+ availableAt: retryAt,
+ }),
+ ).resolves.toBe(true);
+ await expect(methods.markAgentEventActorDetachedActionRunning(update)).resolves.toEqual({
+ status: 'already_applied',
+ });
+ const retryClaim = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-detached-retry',
+ claimToken: 'claim-detached-retry',
+ now: retryAt,
+ leaseUntil: new Date(retryAt.getTime() + 60_000),
+ });
+ const retryAttempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: retryClaim!.id,
+ workerId: 'worker-detached-retry',
+ claimToken: 'claim-detached-retry',
+ now: retryAt,
+ });
+ await expect(
+ methods.deadLetterAgentTriggerDelivery({
+ id: retryClaim!.id,
+ workerId: 'worker-detached-retry',
+ claimToken: 'claim-detached-retry',
+ attempt: retryAttempt!,
+ error: transientFailure({ attemptedAt: retryAt }),
+ settledAt: retryAt,
+ }),
+ ).resolves.toBe(true);
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ ...update,
+ status: 'succeeded',
+ result: 'accepted',
+ observedAt: new Date(generationCreatedAt + 4_000),
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ await expect(
+ methods.getAgentEventActorDetachedAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ }),
+ ).resolves.toEqual(expect.objectContaining({ status: 'succeeded', result: 'accepted' }));
+ });
+
+ it('reserves a detached retry for the exact leased owner before handling is persisted', async () => {
+ const bindingId = 'binding-detached-leased-retry';
+ const conversationId = 'conversation-detached-leased-retry';
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ awaitTerminalHandling: true,
+ tenantId: undefined,
+ envelope: {
+ mode: 'continue',
+ target: { bindingId },
+ event: { source: { id: 'source-key-1', type: 'remote_api_key' } },
+ },
+ }),
+ );
+ await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-detached-leased-retry',
+ claimToken: 'claim-detached-leased-retry',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const generationCreatedAt = START.getTime() + 1_000;
+ await expect(
+ methods.admitAgentEventActorAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ admittedAt: new Date(generationCreatedAt),
+ admissionId: 'admission-detached-leased-retry',
+ }),
+ ).resolves.toBe(true);
+ const first = await methods.reserveAgentEventActorDetachedAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ turnId: 'response-detached-leased-retry:0',
+ invocationId: queued.delivery.deliveryKey,
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-detached-leased-retry-1',
+ reservedAt: new Date(generationCreatedAt + 1_000),
+ recoveryAfter: new Date(generationCreatedAt + 61_000),
+ });
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ taskId: first.action.taskId,
+ idempotencyKey: first.action.idempotencyKey,
+ status: 'failed',
+ error: 'definite adapter failure',
+ observedAt: new Date(generationCreatedAt + 2_000),
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+
+ const retryInput = {
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ turnId: 'response-detached-leased-retry:1',
+ invocationId: queued.delivery.deliveryKey,
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-detached-leased-retry-2',
+ reservedAt: new Date(generationCreatedAt + 3_000),
+ recoveryAfter: new Date(generationCreatedAt + 63_000),
+ };
+ const retry = await methods.reserveAgentEventActorDetachedAction(retryInput);
+
+ expect(retry).toMatchObject({
+ status: 'reserved',
+ action: {
+ launchAttempt: 1,
+ toolCallId: 'call-detached-leased-retry-2',
+ status: 'reserved',
+ },
+ });
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ taskId: retry.action.taskId,
+ idempotencyKey: retry.action.idempotencyKey,
+ status: 'failed',
+ error: 'retry adapter failure',
+ observedAt: new Date(generationCreatedAt + 4_000),
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ await expect(
+ methods.reserveAgentEventActorDetachedAction({
+ ...retryInput,
+ reservedAt: new Date(generationCreatedAt + 5_000),
+ recoveryAfter: new Date(generationCreatedAt + 65_000),
+ }),
+ ).resolves.toMatchObject({
+ status: 'replay',
+ action: {
+ taskId: retry.action.taskId,
+ idempotencyKey: retry.action.idempotencyKey,
+ launchAttempt: 1,
+ status: 'failed',
+ },
+ });
+ const stored = await Delivery.findOne({ deliveryKey: queued.delivery.deliveryKey })
+ .select('+actorDetachedActionHistory')
+ .lean();
+ expect(stored).toMatchObject({
+ status: 'leased',
+ actorDetachedActionHistory: [
+ expect.objectContaining({ taskId: first.action.taskId, status: 'failed' }),
+ ],
+ });
+ expect(stored).not.toHaveProperty('handling');
+ });
+
+ it('gives one replica the detached expected-action launch reservation', async () => {
+ const bindingId = 'binding-detached-1';
+ const conversationId = 'conversation-detached-1';
+ const queued = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ awaitTerminalHandling: true,
+ tenantId: undefined,
+ envelope: {
+ mode: 'continue',
+ target: { bindingId },
+ event: { source: { id: 'source-key-1', type: 'remote_api_key' } },
+ },
+ }),
+ );
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-detached-1',
+ claimToken: 'claim-detached-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: claimed!.id,
+ workerId: 'worker-detached-1',
+ claimToken: 'claim-detached-1',
+ now: START,
+ });
+ const generationCreatedAt = START.getTime() + 1_000;
+ const admission = {
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ admittedAt: new Date(generationCreatedAt),
+ admissionId: 'admission-detached-1',
+ };
+ await expect(methods.admitAgentEventActorAction(admission)).resolves.toBe(true);
+ await methods.completeAgentTriggerDelivery({
+ id: claimed!.id,
+ workerId: 'worker-detached-1',
+ claimToken: 'claim-detached-1',
+ attempt: attempt!,
+ result: { status: 'started' },
+ settledAt: new Date(generationCreatedAt),
+ awaitTerminalHandling: true,
+ handling: {
+ status: 'started',
+ conversationId,
+ streamId: conversationId,
+ generationCreatedAt,
+ startedAt: new Date(generationCreatedAt),
+ },
+ });
+ const reservation = {
+ deliveryKey: queued.delivery.deliveryKey,
+ user: queued.delivery.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ turnId: 'response-detached-1:0',
+ invocationId: queued.delivery.deliveryKey,
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-detached-1',
+ reservedAt: new Date(generationCreatedAt + 1_000),
+ recoveryAfter: new Date(generationCreatedAt + 61_000),
+ };
+
+ const contenders = await Promise.all([
+ methods.reserveAgentEventActorDetachedAction(reservation),
+ methods.reserveAgentEventActorDetachedAction(reservation),
+ ]);
+
+ expect(contenders.map((result) => result.status).sort()).toEqual(['replay', 'reserved']);
+ expect(contenders[0].action).toEqual(contenders[1].action);
+ expect(contenders[0].action).toMatchObject({
+ version: 1,
+ invocationId: reservation.invocationId,
+ expectedToolName: reservation.expectedToolName,
+ toolName: reservation.toolName,
+ toolCallId: reservation.toolCallId,
+ taskId: expect.stringMatching(/^event_actor_[a-f0-9]{64}$/),
+ status: 'reserved',
+ launchAttempt: 0,
+ reservedAt: reservation.reservedAt,
+ });
+ expect(contenders[0].action.idempotencyKey).toMatch(/^[a-f0-9]{64}$/);
+ await expect(methods.reserveAgentEventActorDetachedAction(reservation)).resolves.toEqual({
+ status: 'replay',
+ action: contenders[0].action,
+ });
+
+ const update = {
+ deliveryKey: reservation.deliveryKey,
+ user: reservation.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ taskId: contenders[0].action.taskId,
+ idempotencyKey: contenders[0].action.idempotencyKey,
+ observedAt: new Date(generationCreatedAt + 2_000),
+ recoveryAfter: new Date(generationCreatedAt + 1_802_000),
+ };
+ /** The same durable state covers process death both before invoke and in
+ * the unobservable invoke->acknowledgement window: neither permits relaunch. */
+ await expect(
+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({
+ ...update,
+ observedAt: reservation.recoveryAfter,
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ ...update,
+ status: 'failed',
+ error: 'move service unavailable',
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ await expect(methods.markAgentEventActorDetachedActionRunning(update)).resolves.toEqual({
+ status: 'already_applied',
+ });
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ ...update,
+ observedAt: new Date(generationCreatedAt + 5_000),
+ status: 'failed',
+ error: 'move service unavailable',
+ }),
+ ).resolves.toEqual({ status: 'already_applied' });
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ ...update,
+ status: 'succeeded',
+ result: 'conflicting result',
+ }),
+ ).resolves.toEqual({ status: 'conflict' });
+ await expect(methods.reserveAgentEventActorDetachedAction(reservation)).resolves.toMatchObject({
+ status: 'replay',
+ action: { taskId: contenders[0].action.taskId, status: 'failed' },
+ });
+ const retryReservation = await methods.reserveAgentEventActorDetachedAction({
+ ...reservation,
+ turnId: 'response-detached-1:1',
+ reservedAt: new Date(generationCreatedAt + 3_000),
+ recoveryAfter: new Date(generationCreatedAt + 63_000),
+ });
+ expect(retryReservation).toMatchObject({
+ status: 'reserved',
+ action: {
+ launchAttempt: 1,
+ toolCallId: reservation.toolCallId,
+ status: 'reserved',
+ },
+ });
+ expect(retryReservation.action.taskId).not.toBe(contenders[0].action.taskId);
+ const retryUpdate = {
+ ...update,
+ taskId: retryReservation.action.taskId,
+ idempotencyKey: retryReservation.action.idempotencyKey,
+ observedAt: new Date(generationCreatedAt + 4_000),
+ };
+ await expect(
+ methods.markAgentEventActorDetachedActionRunning({
+ ...retryUpdate,
+ recoveryAfter: new Date(generationCreatedAt + 1_804_000),
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ await expect(
+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({
+ ...retryUpdate,
+ observedAt: new Date(generationCreatedAt + 1_803_999),
+ }),
+ ).resolves.toEqual({ status: 'conflict' });
+ await expect(
+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({
+ ...retryUpdate,
+ observedAt: new Date(generationCreatedAt + 1_804_000),
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ await expect(
+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({
+ ...retryUpdate,
+ observedAt: new Date(generationCreatedAt + 1_805_000),
+ }),
+ ).resolves.toEqual({ status: 'already_applied' });
+ await expect(
+ methods.settleAgentEventActorDetachedAction({
+ ...retryUpdate,
+ observedAt: new Date(generationCreatedAt + 1_806_000),
+ status: 'succeeded',
+ result: 'move accepted',
+ }),
+ ).resolves.toEqual({ status: 'applied' });
+ const stored = await Delivery.findOne({ deliveryKey: reservation.deliveryKey })
+ .select('+actorDetachedAction +actorDetachedActionHistory')
+ .lean();
+ expect(stored?.actorDetachedAction).toMatchObject({
+ status: 'succeeded',
+ result: 'move accepted',
+ launchAttempt: 1,
+ });
+ expect(stored?.actorDetachedActionHistory).toEqual([
+ expect.objectContaining({
+ taskId: contenders[0].action.taskId,
+ status: 'failed',
+ error: 'move service unavailable',
+ }),
+ ]);
+ await expect(
+ methods.getAgentEventActorDetachedAction({
+ deliveryKey: reservation.deliveryKey,
+ user: reservation.user,
+ bindingId,
+ conversationId,
+ generationCreatedAt,
+ }),
+ ).resolves.toMatchObject({
+ taskId: retryReservation.action.taskId,
+ status: 'succeeded',
+ result: 'move accepted',
+ });
+ });
+
it('serializes no-action settlement against renewed action admission', async () => {
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
@@ -2649,10 +3342,11 @@ describe('agent trigger delivery methods', () => {
});
});
- it('counts only live leases while an account deletion drains', async () => {
+ it('counts live leases and detached launch authority while an account deletion drains', async () => {
const user = new mongoose.Types.ObjectId();
- await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
- await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ const leased = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ const detached = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ const admitted = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
await methods.claimNextAgentTriggerDelivery({
workerId: 'worker-1',
claimToken: 'claim-1',
@@ -2660,7 +3354,73 @@ describe('agent trigger delivery methods', () => {
leaseUntil: new Date(START.getTime() + 60_000),
});
- expect(await methods.countActiveAgentTriggerDeliveriesByUser(user, START)).toBe(1);
+ await Delivery.updateOne(
+ { _id: detached.delivery.id },
+ {
+ $set: {
+ status: 'succeeded',
+ actorDetachedAction: {
+ version: 1,
+ invocationId: detached.delivery.deliveryKey,
+ expectedToolName: 'submit_move',
+ toolName: 'submit_move_mcp_chess',
+ toolCallId: 'call-delete-drain',
+ turnId: 'response-delete-drain:0',
+ taskId: `event_actor_${'d'.repeat(64)}`,
+ idempotencyKey: 'd'.repeat(64),
+ launchAttempt: 0,
+ status: 'running',
+ reservedAt: START,
+ observedAt: START,
+ recoveryAfter: new Date(START.getTime() + 60_000),
+ launchedAt: START,
+ },
+ },
+ },
+ );
+ await Delivery.updateOne(
+ { _id: admitted.delivery.id },
+ {
+ $set: {
+ status: 'succeeded',
+ actorActionAdmittedAt: START,
+ actorActionAdmissionId: 'admission-delete-drain',
+ },
+ },
+ );
+
+ expect(leased.delivery.id).not.toBe(detached.delivery.id);
+ expect(await methods.countActiveAgentTriggerDeliveriesByUser(user, START)).toBe(3);
+ expect(
+ await methods.countActiveAgentTriggerDeliveriesByUser(
+ user,
+ new Date(START.getTime() + 60_001),
+ ),
+ ).toBe(2);
+ await Delivery.updateOne(
+ { _id: detached.delivery.id },
+ { $set: { 'actorDetachedAction.status': 'launch_indeterminate' } },
+ );
+ expect(
+ await methods.countActiveAgentTriggerDeliveriesByUser(
+ user,
+ new Date(START.getTime() + 60_001),
+ ),
+ ).toBe(2);
+ await Delivery.updateOne(
+ { _id: detached.delivery.id },
+ { $set: { 'actorDetachedAction.status': 'succeeded' } },
+ );
+ expect(
+ await methods.countActiveAgentTriggerDeliveriesByUser(
+ user,
+ new Date(START.getTime() + 60_001),
+ ),
+ ).toBe(1);
+ await Delivery.updateOne(
+ { _id: admitted.delivery.id },
+ { $unset: { actorActionAdmittedAt: 1, actorActionAdmissionId: 1 } },
+ );
expect(
await methods.countActiveAgentTriggerDeliveriesByUser(
user,
@@ -2669,6 +3429,67 @@ describe('agent trigger delivery methods', () => {
).toBe(0);
});
+ it('serializes account deletion against action admission on each delivery row', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const fenceStartedAt = new Date(START);
+ await User.create({
+ _id: user,
+ email: '[email protected]',
+ provider: 'local',
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ const first = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ user,
+ envelope: { mode: 'continue', target: { bindingId: 'binding-purge' } },
+ }),
+ );
+ const second = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({
+ user,
+ envelope: { mode: 'continue', target: { bindingId: 'binding-purge' } },
+ }),
+ );
+ await Delivery.updateMany({ user }, { $set: { status: 'succeeded' } });
+ const admission = {
+ deliveryKey: first.delivery.deliveryKey,
+ user,
+ tenantId: 'tenant-1',
+ bindingId: 'binding-purge',
+ conversationId: 'conversation-purge',
+ admittedAt: START,
+ admissionId: 'admission-before-purge',
+ };
+
+ await expect(methods.admitAgentEventActorAction(admission)).resolves.toBe(true);
+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);
+ await expect(
+ methods.countActiveAgentTriggerDeliveriesByUser(user, new Date(START.getTime() + 60_001)),
+ ).resolves.toBe(1);
+ await expect(
+ methods.admitAgentEventActorAction({
+ ...admission,
+ deliveryKey: second.delivery.deliveryKey,
+ admissionId: 'admission-after-purge',
+ }),
+ ).resolves.toBe(false);
+ await expect(
+ Delivery.countDocuments({ user, actorActionAdmissionClosedAt: fenceStartedAt }),
+ ).resolves.toBe(2);
+
+ await expect(methods.cancelAgentTriggerUserPurge(user, fenceStartedAt)).resolves.toBe(true);
+ await expect(
+ Delivery.countDocuments({ user, actorActionAdmissionClosedAt: { $exists: true } }),
+ ).resolves.toBe(0);
+ await expect(
+ methods.admitAgentEventActorAction({
+ ...admission,
+ deliveryKey: second.delivery.deliveryKey,
+ admissionId: 'admission-after-cancel',
+ }),
+ ).resolves.toBe(true);
+ });
+
it('deletes all queued payloads for an erased user', async () => {
const user = new mongoose.Types.ObjectId();
const fenceStartedAt = new Date(START);
@@ -2758,6 +3579,9 @@ describe('agent trigger delivery methods', () => {
await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(0);
expect(await Delivery.countDocuments({ user })).toBe(1);
+ expect(
+ await Delivery.countDocuments({ user, actorActionAdmissionClosedAt: { $exists: true } }),
+ ).toBe(0);
expect(await UserPurge.countDocuments({ _id: user })).toBe(0);
});
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
index d300d62191..c507276d1a 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -1,5 +1,7 @@
+import { createHash } from 'node:crypto';
import type { Model, Types } from 'mongoose';
import type {
+ AgentEventActorDetachedAction,
AgentTriggerDeliveryClaim,
AgentEventActorReceipt,
AgentTriggerDeliveryFailure,
@@ -14,6 +16,7 @@ import type {
IAgentTriggerUserPurge,
IAgentTriggerUserPurgeDocument,
} from '~/types/triggerDelivery';
+import { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from '~/types/triggerDelivery';
import { createIndexesWithRetry } from '~/utils/retry';
import logger from '~/config/winston';
@@ -29,6 +32,10 @@ const HISTORY_LIMIT = 64;
const MAX_BATCH_SIZE = 8;
const MAX_BATCH_BYTES = 512 * 1024;
+function isStagingStatus(status: IAgentTriggerDelivery['status']): boolean {
+ return status === 'staging' || status === 'capability_staging';
+}
+
type DuplicateKeyError = { code?: number };
export type AgentEventActorReceiptMetric = {
@@ -75,6 +82,7 @@ export interface EnqueueAgentTriggerDeliveryInput {
coalesceFrom?: Date;
coalesceUntil?: Date;
awaitTerminalHandling?: boolean;
+ requiredWorkerCapability?: string;
}
export interface AgentTriggerDeliveryFence {
@@ -118,6 +126,42 @@ export interface AgentEventActorActionAdmissionInput extends GetAgentEventActorR
admissionId: string;
}
+/** Durable transition receipt. Storage unavailability remains an exception;
+ * callers never collapse it into a conflict or a negative acknowledgement. */
+export interface AgentEventActorDetachedTransitionResult {
+ status: 'applied' | 'already_applied' | 'conflict';
+}
+
+export interface ReserveAgentEventActorDetachedActionInput extends GetAgentEventActorReceiptInput {
+ generationCreatedAt: number;
+ turnId: string;
+ invocationId: string;
+ expectedToolName: string;
+ toolName: string;
+ toolCallId: string;
+ reservedAt: Date;
+ recoveryAfter: Date;
+}
+
+export interface UpdateAgentEventActorDetachedActionInput extends GetAgentEventActorReceiptInput {
+ generationCreatedAt: number;
+ taskId: string;
+ idempotencyKey: string;
+ observedAt: Date;
+}
+
+export interface MarkAgentEventActorDetachedActionRunningInput
+ extends UpdateAgentEventActorDetachedActionInput {
+ recoveryAfter: Date;
+}
+
+export interface SettleAgentEventActorDetachedActionInput
+ extends UpdateAgentEventActorDetachedActionInput {
+ status: 'succeeded' | 'failed' | 'cancelled';
+ result?: string;
+ error?: string;
+}
+
export interface GetAgentEventActorReceiptInput {
deliveryKey: string;
user: string | Types.ObjectId;
@@ -145,6 +189,7 @@ export interface AgentTriggerDeliveryMethods {
claimToken: string;
now: Date;
leaseUntil: Date;
+ workerCapabilities?: string[];
}) => Promise<AgentTriggerDeliveryClaim | null>;
findEarlierAgentTriggerDelivery: (
delivery: Pick<AgentTriggerDeliveryRecord, 'orderingKey' | 'laneSequence'>,
@@ -181,6 +226,24 @@ export interface AgentTriggerDeliveryMethods {
hasAgentEventActorActionAdmission: (
input: AgentEventActorActionAdmissionInput,
) => Promise<boolean>;
+ reserveAgentEventActorDetachedAction: (
+ input: ReserveAgentEventActorDetachedActionInput,
+ ) => Promise<{
+ status: 'reserved' | 'replay' | 'conflict';
+ action: AgentEventActorDetachedAction;
+ }>;
+ markAgentEventActorDetachedActionRunning: (
+ input: MarkAgentEventActorDetachedActionRunningInput,
+ ) => Promise<AgentEventActorDetachedTransitionResult>;
+ markAgentEventActorDetachedActionLaunchIndeterminate: (
+ input: UpdateAgentEventActorDetachedActionInput,
+ ) => Promise<AgentEventActorDetachedTransitionResult>;
+ settleAgentEventActorDetachedAction: (
+ input: SettleAgentEventActorDetachedActionInput,
+ ) => Promise<AgentEventActorDetachedTransitionResult>;
+ getAgentEventActorDetachedAction: (
+ input: GetAgentEventActorReceiptInput & { generationCreatedAt: number },
+ ) => Promise<AgentEventActorDetachedAction | null>;
settleAgentEventActorReceipt: (input: SettleAgentEventActorReceiptInput) => Promise<boolean>;
getAgentEventActorReceipt: (
input: GetAgentEventActorReceiptInput,
@@ -260,6 +323,9 @@ function toRecord(delivery: IAgentTriggerDelivery): AgentTriggerDeliveryRecord {
availableAt: delivery.availableAt,
createdAt: delivery.createdAt,
...(delivery.tenantId != null && { tenantId: delivery.tenantId }),
+ ...(delivery.requiredWorkerCapability != null && {
+ requiredWorkerCapability: delivery.requiredWorkerCapability,
+ }),
...(delivery.leaseBy != null && { leaseBy: delivery.leaseBy }),
...(delivery.leaseUntil != null && { leaseUntil: delivery.leaseUntil }),
...(delivery.claimToken != null && { claimToken: delivery.claimToken }),
@@ -295,7 +361,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliv
}
const record = toRecord(delivery);
if (
- record.status !== 'leased' ||
+ !['leased', 'capability_leased'].includes(record.status) ||
record.claimToken == null ||
record.leaseBy == null ||
record.leaseUntil == null
@@ -335,6 +401,9 @@ export function createAgentTriggerDeliveryMethods(
_id: lane._id,
value: lane.value,
publisherDeliveryId,
+ ...(lane.publisherRequeueCount == null
+ ? { publisherRequeueCount: { $exists: false } }
+ : { publisherRequeueCount: lane.publisherRequeueCount }),
};
if (lane.value === 1) {
const deleted = await LaneSequence().deleteOne(publisherFence);
@@ -343,7 +412,7 @@ export function createAgentTriggerDeliveryMethods(
const released = await LaneSequence().updateOne(publisherFence, {
$inc: { value: -1 },
$set: { cleanupRequestedAt: new Date() },
- $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },
+ $unset: { publisherDeliveryId: 1, publisherRequeueCount: 1, publisherStartedAt: 1 },
});
return released.modifiedCount === 1;
}
@@ -358,6 +427,26 @@ export function createAgentTriggerDeliveryMethods(
}
const staged = await Delivery().findById(publisherDeliveryId).lean<IAgentTriggerDelivery>();
+ const publicationGeneration = lane.publisherRequeueCount ?? staged?.requeueCount ?? 0;
+ if (lane.publisherRequeueCount == null) {
+ /** Upgrade a publisher acquired by a pre-generation binary before
+ * touching its delivery. The exact lane fence prevents a stale helper
+ * from adopting the requeue generation of a later publisher. */
+ const adopted = await LaneSequence().updateOne(
+ {
+ _id: lane._id,
+ value: lane.value,
+ publisherDeliveryId,
+ publisherRequeueCount: { $exists: false },
+ },
+ { $set: { publisherRequeueCount: publicationGeneration } },
+ { timestamps: false },
+ );
+ if (adopted.modifiedCount !== 1) {
+ return false;
+ }
+ }
+ const publicationLane = { ...lane, publisherRequeueCount: publicationGeneration };
let batchRoot: IAgentTriggerDelivery | null = null;
if (
staged?._id != null &&
@@ -432,11 +521,22 @@ export function createAgentTriggerDeliveryMethods(
batchRoot.awaitTerminalHandling = true;
}
}
+ let publishedStatus: IAgentTriggerDelivery['status'] = 'pending';
+ if (batchRoot != null) {
+ publishedStatus = 'batched';
+ } else if (staged?.requiredWorkerCapability != null) {
+ publishedStatus = 'capability_pending';
+ }
const published = await Delivery().updateOne(
- { _id: publisherDeliveryId, orderingKey: lane._id, status: 'staging' },
+ {
+ _id: publisherDeliveryId,
+ orderingKey: lane._id,
+ status: { $in: ['staging', 'capability_staging'] },
+ requeueCount: publicationGeneration,
+ },
{
$set: {
- status: batchRoot == null ? 'pending' : 'batched',
+ status: publishedStatus,
laneSequence: lane.value,
...(batchRoot?._id != null && {
batchRootId: batchRoot._id,
@@ -450,19 +550,27 @@ export function createAgentTriggerDeliveryMethods(
if (published.modifiedCount === 0) {
let current = await Delivery()
.findById(publisherDeliveryId)
- .select('orderingKey laneSequence status batchRootId')
+ .select('orderingKey laneSequence status batchRootId requeueCount')
.lean<
- Pick<IAgentTriggerDelivery, 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId'>
+ Pick<
+ IAgentTriggerDelivery,
+ 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId' | 'requeueCount'
+ >
>();
if (
batchRoot?._id != null &&
current != null &&
current.orderingKey === lane._id &&
- current.status !== 'staging' &&
+ !isStagingStatus(current.status) &&
current.laneSequence !== lane.value
) {
await Delivery().updateOne(
- { _id: publisherDeliveryId, orderingKey: lane._id, status: current.status },
+ {
+ _id: publisherDeliveryId,
+ orderingKey: lane._id,
+ status: current.status,
+ requeueCount: publicationGeneration,
+ },
{
$set: {
laneSequence: lane.value,
@@ -473,17 +581,24 @@ export function createAgentTriggerDeliveryMethods(
);
current = await Delivery()
.findById(publisherDeliveryId)
- .select('orderingKey laneSequence status batchRootId')
+ .select('orderingKey laneSequence status batchRootId requeueCount')
.lean<
- Pick<IAgentTriggerDelivery, 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId'>
+ Pick<
+ IAgentTriggerDelivery,
+ 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId' | 'requeueCount'
+ >
>();
}
publicationCommitted =
current != null &&
current.orderingKey === lane._id &&
- current.status !== 'staging' &&
+ !isStagingStatus(current.status) &&
+ current.requeueCount === publicationGeneration &&
current.laneSequence === lane.value;
- if (current?.orderingKey === lane._id && current.status === 'staging') {
+ if (current?.orderingKey === lane._id && isStagingStatus(current.status)) {
+ if (current.requeueCount !== publicationGeneration) {
+ return abandonLanePublisher(publicationLane);
+ }
throw new Error('Failed to publish the reserved agent trigger delivery');
}
}
@@ -492,11 +607,12 @@ export function createAgentTriggerDeliveryMethods(
_id: lane._id,
value: lane.value,
publisherDeliveryId,
+ publisherRequeueCount: publicationGeneration,
};
if (publicationCommitted) {
const released = await LaneSequence().updateOne(publisherFence, {
$set: { tailDeliveryId: publisherDeliveryId },
- $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },
+ $unset: { publisherDeliveryId: 1, publisherRequeueCount: 1, publisherStartedAt: 1 },
});
return published.modifiedCount === 1 || released.modifiedCount === 1;
}
@@ -505,7 +621,7 @@ export function createAgentTriggerDeliveryMethods(
// reservation was acquired (or was removed by account cleanup). No later
// reservation can exist while this fence is held, so the unused sequence
// can be rolled back without creating a gap or regressing the lane tail.
- return abandonLanePublisher(lane);
+ return abandonLanePublisher(publicationLane);
}
/** A dead batch root owns requeue before any constituent is reset. Staging
@@ -572,7 +688,7 @@ export function createAgentTriggerDeliveryMethods(
if (current == null) {
throw new Error('Staged agent trigger delivery disappeared before publication');
}
- if (current.status !== 'staging') {
+ if (!isStagingStatus(current.status)) {
return current;
}
if ((await UserPurge().exists({ _id: current.user })) != null) {
@@ -593,7 +709,7 @@ export function createAgentTriggerDeliveryMethods(
// the oldest visible row first lets any replica repair a writer that
// stopped in that gap without allowing a later enqueue to overtake it.
const next = await Delivery()
- .findOne({ orderingKey, status: 'staging' })
+ .findOne({ orderingKey, status: { $in: ['staging', 'capability_staging'] } })
// updatedAt is the current staging admission time: initial enqueue
// sets it on insert, while explicit requeue refreshes it so an old
// dead letter is admitted behind staging work that already exists.
@@ -612,6 +728,7 @@ export function createAgentTriggerDeliveryMethods(
$inc: { value: 1 },
$set: {
publisherDeliveryId: next._id,
+ publisherRequeueCount: next.requeueCount ?? 0,
publisherStartedAt: new Date(),
},
$unset: { cleanupRequestedAt: 1 },
@@ -647,6 +764,15 @@ export function createAgentTriggerDeliveryMethods(
async function enqueueAgentTriggerDelivery(
input: EnqueueAgentTriggerDeliveryInput,
): Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }> {
+ if (
+ input.requiredWorkerCapability != null &&
+ input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1
+ ) {
+ throw new TypeError('Agent trigger delivery requires an unsupported worker capability');
+ }
+ if (input.requiredWorkerCapability != null && input.coalesceKey != null) {
+ throw new TypeError('Capability-fenced agent trigger deliveries cannot be batched');
+ }
let staged: IAgentTriggerDelivery;
let replayed = false;
try {
@@ -660,7 +786,7 @@ export function createAgentTriggerDeliveryMethods(
batchMemberIds: [],
}),
laneSequence: 0,
- status: 'staging',
+ status: input.requiredWorkerCapability == null ? 'staging' : 'capability_staging',
attempts: 0,
requeueCount: 0,
stagingRecoveryAt: new Date(),
@@ -679,7 +805,7 @@ export function createAgentTriggerDeliveryMethods(
if (existing.fingerprint !== input.fingerprint) {
throw new AgentTriggerDeliveryConflictError(input.deliveryKey);
}
- if (existing.status !== 'staging') {
+ if (!isStagingStatus(existing.status)) {
return { delivery: toRecord(existing), replayed: true };
}
staged = existing;
@@ -716,6 +842,9 @@ export function createAgentTriggerDeliveryMethods(
_id: lane._id,
value: lane.value,
publisherDeliveryId: lane.publisherDeliveryId,
+ ...(lane.publisherRequeueCount == null
+ ? { publisherRequeueCount: { $exists: false } }
+ : { publisherRequeueCount: lane.publisherRequeueCount }),
publisherStartedAt: lane.publisherStartedAt,
},
{ $set: { publisherStartedAt: recoveryCursor } },
@@ -736,7 +865,10 @@ export function createAgentTriggerDeliveryMethods(
/** Pre-cursor replicas can leave staging rows outside the sparse recovery
* index. Backfill them in bounded batches before using the indexed scan. */
const legacyStaged = await Delivery()
- .find({ status: 'staging', stagingRecoveryAt: { $exists: false } })
+ .find({
+ status: { $in: ['staging', 'capability_staging'] },
+ stagingRecoveryAt: { $exists: false },
+ })
.sort({ updatedAt: 1, _id: 1 })
.limit(remaining)
.lean<IAgentTriggerDelivery[]>();
@@ -744,7 +876,11 @@ export function createAgentTriggerDeliveryMethods(
for (const delivery of legacyStaged) {
if (delivery._id != null) {
await Delivery().updateOne(
- { _id: delivery._id, status: 'staging', stagingRecoveryAt: { $exists: false } },
+ {
+ _id: delivery._id,
+ status: { $in: ['staging', 'capability_staging'] },
+ stagingRecoveryAt: { $exists: false },
+ },
{ $set: { stagingRecoveryAt: legacyCursor } },
{ timestamps: false },
);
@@ -758,7 +894,7 @@ export function createAgentTriggerDeliveryMethods(
remaining > 0
? await Delivery()
.find({
- status: 'staging',
+ status: { $in: ['staging', 'capability_staging'] },
stagingRecoveryAt: { $exists: true },
...(legacyIds.length > 0 && { _id: { $nin: legacyIds } }),
})
@@ -778,7 +914,7 @@ export function createAgentTriggerDeliveryMethods(
await Delivery().updateOne(
{
_id: delivery._id,
- status: 'staging',
+ status: { $in: ['staging', 'capability_staging'] },
stagingRecoveryAt: delivery.stagingRecoveryAt,
},
{ $set: { stagingRecoveryAt: stagingRecoveryCursor } },
@@ -788,7 +924,7 @@ export function createAgentTriggerDeliveryMethods(
for (const delivery of staged) {
try {
const published = await publishStagedDelivery(delivery);
- if (published.status !== 'staging') {
+ if (!isStagingStatus(published.status)) {
recovered += 1;
} else {
await rotateStagingRecovery(delivery);
@@ -815,7 +951,21 @@ export function createAgentTriggerDeliveryMethods(
const stillRetained = await Delivery().exists({
orderingKey,
$or: [
- { status: { $in: ['staging', 'batched', 'pending', 'leased', 'dead'] } },
+ {
+ status: {
+ $in: [
+ 'staging',
+ 'capability_staging',
+ 'batched',
+ 'pending',
+ 'capability_pending',
+ 'leased',
+ 'capability_leased',
+ 'capability_dead',
+ 'dead',
+ ],
+ },
+ },
{
status: 'succeeded',
batchRootId: { $exists: false },
@@ -908,25 +1058,56 @@ export function createAgentTriggerDeliveryMethods(
claimToken: string;
now: Date;
leaseUntil: Date;
+ workerCapabilities?: string[];
}): Promise<AgentTriggerDeliveryClaim | null> {
const expiredBefore = new Date(input.now.getTime() - LEASE_SKEW_MARGIN_MS);
+ const workerCapabilities = [...new Set(input.workerCapabilities ?? [])];
+ if (workerCapabilities.some((value) => value.length === 0 || value.length > 128)) {
+ throw new TypeError('Agent trigger worker capability is invalid');
+ }
+ const capabilityConditions =
+ workerCapabilities.length === 0
+ ? []
+ : [
+ {
+ requiredWorkerCapability: { $in: workerCapabilities },
+ status: 'capability_pending',
+ availableAt: { $lte: input.now },
+ },
+ {
+ requiredWorkerCapability: { $in: workerCapabilities },
+ status: 'capability_leased',
+ leaseUntil: { $lte: expiredBefore },
+ },
+ ];
const claimed = await Delivery()
.findOneAndUpdate(
{
$or: [
{ status: 'pending', availableAt: { $lte: input.now } },
{ status: 'leased', leaseUntil: { $lte: expiredBefore } },
+ ...capabilityConditions,
],
},
- {
- $set: {
- status: 'leased',
- leaseBy: input.workerId,
- leaseUntil: input.leaseUntil,
- claimToken: input.claimToken,
+ [
+ {
+ $set: {
+ status: {
+ $cond: [
+ { $in: ['$status', ['capability_pending', 'capability_leased']] },
+ 'capability_leased',
+ 'leased',
+ ],
+ },
+ leaseBy: input.workerId,
+ leaseUntil: input.leaseUntil,
+ claimToken: input.claimToken,
+ settledAt: '$$REMOVE',
+ expiresAt: '$$REMOVE',
+ updatedAt: input.now,
+ },
},
- $unset: { settledAt: 1, expiresAt: 1 },
- },
+ ],
{ new: true, sort: { availableAt: 1, createdAt: 1, _id: 1 } },
)
.lean<IAgentTriggerDelivery>();
@@ -941,7 +1122,11 @@ export function createAgentTriggerDeliveryMethods(
orderingKey: delivery.orderingKey,
laneSequence: { $lt: delivery.laneSequence },
$or: [
- { status: { $in: ['pending', 'leased'] } },
+ {
+ status: {
+ $in: ['pending', 'capability_pending', 'leased', 'capability_leased'],
+ },
+ },
{
status: 'succeeded',
batchRootId: { $exists: false },
@@ -1205,14 +1390,27 @@ export function createAgentTriggerDeliveryMethods(
const fence = (input: AgentTriggerDeliveryFence) => ({
_id: input.id,
- status: 'leased',
+ status: { $in: ['leased', 'capability_leased'] },
leaseBy: input.workerId,
claimToken: input.claimToken,
});
+ const capabilityFence = (input: AgentTriggerDeliveryFence) => ({
+ ...fence(input),
+ status: 'capability_leased',
+ requiredWorkerCapability: { $exists: true },
+ });
+
async function releaseAgentTriggerDelivery(
input: AgentTriggerDeliveryFence & { availableAt: Date },
): Promise<boolean> {
+ const capabilityResult = await Delivery().updateOne(capabilityFence(input), {
+ $set: { status: 'capability_pending', availableAt: input.availableAt },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
+ });
+ if (capabilityResult.modifiedCount === 1) {
+ return true;
+ }
const result = await Delivery().updateOne(fence(input), {
$set: { status: 'pending', availableAt: input.availableAt },
$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
@@ -1241,14 +1439,25 @@ export function createAgentTriggerDeliveryMethods(
if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) {
throw new TypeError('attempt must be a positive integer');
}
- const result = await Delivery().updateOne(
- { ...fence(input), attempts: input.attempt },
+ const update = {
+ $inc: { attempts: -1 },
+ $set: { availableAt: input.availableAt },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
+ };
+ const capabilityResult = await Delivery().updateOne(
+ { ...capabilityFence(input), attempts: input.attempt },
{
- $inc: { attempts: -1 },
- $set: { status: 'pending', availableAt: input.availableAt },
- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
+ ...update,
+ $set: { ...update.$set, status: 'capability_pending' },
},
);
+ if (capabilityResult.modifiedCount === 1) {
+ return true;
+ }
+ const result = await Delivery().updateOne(
+ { ...fence(input), attempts: input.attempt },
+ { ...update, $set: { ...update.$set, status: 'pending' } },
+ );
return result.modifiedCount === 1;
}
@@ -1491,6 +1700,10 @@ export function createAgentTriggerDeliveryMethods(
},
$unset: {
lastError: 1,
+ ...(input.requiresActionAdmission === true && {
+ actorActionAdmittedAt: 1,
+ actorActionAdmissionId: 1,
+ }),
...(applied && { 'handling.error': 1 }),
...(!applied && { 'handling.action': 1 }),
},
@@ -1534,7 +1747,13 @@ export function createAgentTriggerDeliveryMethods(
$max: {
expiresAt: new Date(actorReceipt.settledAt.getTime() + SUCCESS_RETENTION_MS),
},
- $unset: { lastError: 1 },
+ $unset: {
+ lastError: 1,
+ ...(input.requiresActionAdmission === true && {
+ actorActionAdmittedAt: 1,
+ actorActionAdmissionId: 1,
+ }),
+ },
},
{ new: true },
)
@@ -1598,6 +1817,7 @@ export function createAgentTriggerDeliveryMethods(
'envelope.target.bindingId': input.bindingId,
actorReceipt: { $exists: false },
actorActionAdmittedAt: { $exists: false },
+ actorActionAdmissionClosedAt: { $exists: false },
$or: [
{ handling: { $exists: false } },
{
@@ -1689,6 +1909,333 @@ export function createAgentTriggerDeliveryMethods(
: null;
}
+ /** Atomically gives one replica launch authority for an exact detached expected action. */
+ async function reserveAgentEventActorDetachedAction(
+ input: ReserveAgentEventActorDetachedActionInput,
+ ): Promise<{
+ status: 'reserved' | 'replay' | 'conflict';
+ action: AgentEventActorDetachedAction;
+ }> {
+ const requiredIdentity = [
+ String(input.user),
+ input.deliveryKey,
+ input.bindingId,
+ input.conversationId,
+ String(input.generationCreatedAt),
+ input.turnId,
+ input.invocationId,
+ input.expectedToolName,
+ input.toolName,
+ input.toolCallId,
+ ];
+ const identity = [String(input.user), input.tenantId ?? '', ...requiredIdentity.slice(1)];
+ if (
+ requiredIdentity.some((value) => value.length === 0) ||
+ !Number.isSafeInteger(input.generationCreatedAt) ||
+ input.generationCreatedAt < 0 ||
+ input.turnId.length > 512 ||
+ Number.isNaN(input.reservedAt.getTime()) ||
+ Number.isNaN(input.recoveryAfter.getTime()) ||
+ input.recoveryAfter <= input.reservedAt
+ ) {
+ throw new TypeError('Detached event actor action identity is invalid');
+ }
+ const buildAction = (launchAttempt: number): AgentEventActorDetachedAction => {
+ const idempotencyKey = createHash('sha256')
+ .update([...identity, String(launchAttempt)].join('\0'))
+ .digest('hex');
+ return {
+ version: 1,
+ invocationId: input.invocationId,
+ expectedToolName: input.expectedToolName,
+ toolName: input.toolName,
+ toolCallId: input.toolCallId,
+ turnId: input.turnId,
+ /** Public task handles must not disclose the adapter idempotency secret. */
+ taskId: `event_actor_${createHash('sha256')
+ .update('librechat:event-actor:task:v1\0')
+ .update(idempotencyKey)
+ .digest('hex')}`,
+ idempotencyKey,
+ launchAttempt,
+ status: 'reserved',
+ reservedAt: input.reservedAt,
+ observedAt: input.reservedAt,
+ recoveryAfter: input.recoveryAfter,
+ };
+ };
+ const action = buildAction(0);
+ const tenantScope =
+ input.tenantId == null ? { tenantId: { $exists: false } } : { tenantId: input.tenantId };
+ const reservationOwner = {
+ $or: [
+ {
+ status: { $in: ['leased', 'capability_leased'] },
+ handling: { $exists: false },
+ },
+ {
+ status: { $in: ['succeeded', 'dead'] },
+ 'handling.status': 'started',
+ 'handling.conversationId': input.conversationId,
+ 'handling.generationCreatedAt': input.generationCreatedAt,
+ },
+ ],
+ };
+ const reserved = await Delivery()
+ .findOneAndUpdate(
+ {
+ deliveryKey: input.deliveryKey,
+ user: input.user,
+ ...tenantScope,
+ 'envelope.target.bindingId': input.bindingId,
+ ...reservationOwner,
+ actorReceipt: { $exists: false },
+ actorActionAdmittedAt: { $exists: true },
+ actorDetachedAction: { $exists: false },
+ },
+ { $set: { actorDetachedAction: action } },
+ { new: true },
+ )
+ .select('+actorDetachedAction')
+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();
+ if (reserved?.actorDetachedAction != null) {
+ return { status: 'reserved', action: reserved.actorDetachedAction };
+ }
+ const existing = await Delivery()
+ .findOne({
+ deliveryKey: input.deliveryKey,
+ user: input.user,
+ ...tenantScope,
+ 'envelope.target.bindingId': input.bindingId,
+ ...reservationOwner,
+ })
+ .select('+actorDetachedAction')
+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();
+ if (existing?.actorDetachedAction == null) {
+ throw new Error('Detached event actor action reservation owner is unavailable');
+ }
+ const sameTurn = existing.actorDetachedAction.turnId === input.turnId;
+ const sameLogicalAction =
+ sameTurn &&
+ existing.actorDetachedAction.invocationId === input.invocationId &&
+ existing.actorDetachedAction.expectedToolName === input.expectedToolName &&
+ existing.actorDetachedAction.toolName === input.toolName &&
+ existing.actorDetachedAction.toolCallId === input.toolCallId;
+ if (sameLogicalAction) {
+ return { status: 'replay', action: existing.actorDetachedAction };
+ }
+ if (sameTurn) {
+ return { status: 'conflict', action: existing.actorDetachedAction };
+ }
+ if (existing.actorDetachedAction.idempotencyKey === action.idempotencyKey) {
+ return { status: 'replay', action: existing.actorDetachedAction };
+ }
+ if (
+ !['failed', 'cancelled'].includes(existing.actorDetachedAction.status) ||
+ existing.actorDetachedAction.launchAttempt >= 15
+ ) {
+ return { status: 'conflict', action: existing.actorDetachedAction };
+ }
+ const retryAction = buildAction(existing.actorDetachedAction.launchAttempt + 1);
+ const retried = await Delivery()
+ .findOneAndUpdate(
+ {
+ deliveryKey: input.deliveryKey,
+ user: input.user,
+ ...tenantScope,
+ 'envelope.target.bindingId': input.bindingId,
+ ...reservationOwner,
+ actorReceipt: { $exists: false },
+ actorActionAdmittedAt: { $exists: true },
+ 'actorDetachedAction.taskId': existing.actorDetachedAction.taskId,
+ 'actorDetachedAction.idempotencyKey': existing.actorDetachedAction.idempotencyKey,
+ 'actorDetachedAction.status': existing.actorDetachedAction.status,
+ },
+ {
+ $set: { actorDetachedAction: retryAction },
+ $push: {
+ actorDetachedActionHistory: {
+ $each: [existing.actorDetachedAction],
+ $slice: -8,
+ },
+ },
+ },
+ { new: true },
+ )
+ .select('+actorDetachedAction')
+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();
+ if (retried?.actorDetachedAction != null) {
+ return { status: 'reserved', action: retried.actorDetachedAction };
+ }
+ const winner = await Delivery()
+ .findOne({
+ deliveryKey: input.deliveryKey,
+ user: input.user,
+ ...tenantScope,
+ 'envelope.target.bindingId': input.bindingId,
+ ...reservationOwner,
+ })
+ .select('+actorDetachedAction')
+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();
+ if (winner?.actorDetachedAction == null) {
+ throw new Error('Detached event actor retry reservation owner is unavailable');
+ }
+ return {
+ status:
+ winner.actorDetachedAction.idempotencyKey === retryAction.idempotencyKey
+ ? 'replay'
+ : 'conflict',
+ action: winner.actorDetachedAction,
+ };
+ }
+
+ function detachedActionScope(input: UpdateAgentEventActorDetachedActionInput) {
+ return {
+ deliveryKey: input.deliveryKey,
+ user: input.user,
+ ...(input.tenantId == null ? { tenantId: { $exists: false } } : { tenantId: input.tenantId }),
+ 'envelope.target.bindingId': input.bindingId,
+ actorReceipt: { $exists: false },
+ actorActionAdmittedAt: { $exists: true },
+ 'actorDetachedAction.taskId': input.taskId,
+ 'actorDetachedAction.idempotencyKey': input.idempotencyKey,
+ };
+ }
+
+ /** Acknowledges launch without regressing terminal evidence from a fast completion. */
+ async function markAgentEventActorDetachedActionRunning(
+ input: MarkAgentEventActorDetachedActionRunningInput,
+ ): Promise<AgentEventActorDetachedTransitionResult> {
+ if (
+ Number.isNaN(input.observedAt.getTime()) ||
+ Number.isNaN(input.recoveryAfter.getTime()) ||
+ input.recoveryAfter <= input.observedAt
+ ) {
+ throw new TypeError('Detached event actor running lease is invalid');
+ }
+ const scope = detachedActionScope(input);
+ const running = await Delivery().updateOne(
+ { ...scope, 'actorDetachedAction.status': 'reserved' },
+ {
+ $set: {
+ 'actorDetachedAction.status': 'running',
+ 'actorDetachedAction.launchedAt': input.observedAt,
+ 'actorDetachedAction.observedAt': input.observedAt,
+ 'actorDetachedAction.recoveryAfter': input.recoveryAfter,
+ },
+ },
+ );
+ if (running.modifiedCount === 1) {
+ return { status: 'applied' };
+ }
+ const alreadyApplied =
+ (await Delivery().exists({
+ ...scope,
+ 'actorDetachedAction.status': {
+ $in: ['running', 'succeeded', 'failed', 'cancelled'],
+ },
+ })) != null;
+ return { status: alreadyApplied ? 'already_applied' : 'conflict' };
+ }
+
+ /** Converts an expired executor lease into durable uncertainty. This state
+ * deliberately cannot be retried; only exact terminal proof may close it. */
+ async function markAgentEventActorDetachedActionLaunchIndeterminate(
+ input: UpdateAgentEventActorDetachedActionInput,
+ ): Promise<AgentEventActorDetachedTransitionResult> {
+ if (Number.isNaN(input.observedAt.getTime())) {
+ throw new TypeError('observedAt must be a valid date');
+ }
+ const scope = detachedActionScope(input);
+ const marked = await Delivery().updateOne(
+ {
+ ...scope,
+ 'actorDetachedAction.status': { $in: ['reserved', 'running'] },
+ 'actorDetachedAction.recoveryAfter': { $lte: input.observedAt },
+ },
+ {
+ $set: {
+ 'actorDetachedAction.status': 'launch_indeterminate',
+ 'actorDetachedAction.observedAt': input.observedAt,
+ },
+ },
+ );
+ if (marked.modifiedCount === 1) {
+ return { status: 'applied' };
+ }
+ const alreadyApplied =
+ (await Delivery().exists({
+ ...scope,
+ 'actorDetachedAction.status': 'launch_indeterminate',
+ })) != null;
+ return { status: alreadyApplied ? 'already_applied' : 'conflict' };
+ }
+
+ /** Persists exact terminal evidence once; identical callbacks replay safely. */
+ async function settleAgentEventActorDetachedAction(
+ input: SettleAgentEventActorDetachedActionInput,
+ ): Promise<AgentEventActorDetachedTransitionResult> {
+ if (
+ Number.isNaN(input.observedAt.getTime()) ||
+ (input.result != null && input.result.length > 32_768) ||
+ (input.error != null && input.error.length > 2_048)
+ ) {
+ throw new TypeError('Detached event actor terminal evidence is invalid');
+ }
+ if (input.status === 'succeeded' ? input.error != null : input.result != null) {
+ throw new TypeError('Detached event actor terminal evidence conflicts with its status');
+ }
+ const terminal = {
+ 'actorDetachedAction.status': input.status,
+ 'actorDetachedAction.settledAt': input.observedAt,
+ 'actorDetachedAction.observedAt': input.observedAt,
+ ...(input.result == null ? {} : { 'actorDetachedAction.result': input.result }),
+ ...(input.error == null ? {} : { 'actorDetachedAction.error': input.error }),
+ };
+ const scope = detachedActionScope(input);
+ const settled = await Delivery().updateOne(
+ {
+ ...scope,
+ 'actorDetachedAction.status': {
+ $in: ['reserved', 'running', 'launch_indeterminate'],
+ },
+ },
+ { $set: terminal },
+ );
+ if (settled.modifiedCount === 1) {
+ return { status: 'applied' };
+ }
+ /** Callback observation time is transport metadata, not terminal identity.
+ * A retry may arrive later; match the exact durable outcome while retaining
+ * the first settlement timestamp. */
+ const alreadyApplied =
+ (await Delivery().exists({
+ ...scope,
+ 'actorDetachedAction.status': input.status,
+ 'actorDetachedAction.result': input.result == null ? { $exists: false } : input.result,
+ 'actorDetachedAction.error': input.error == null ? { $exists: false } : input.error,
+ })) != null;
+ return { status: alreadyApplied ? 'already_applied' : 'conflict' };
+ }
+
+ async function getAgentEventActorDetachedAction(
+ input: GetAgentEventActorReceiptInput & { generationCreatedAt: number },
+ ): Promise<AgentEventActorDetachedAction | null> {
+ const delivery = await Delivery()
+ .findOne({
+ deliveryKey: input.deliveryKey,
+ user: input.user,
+ ...(input.tenantId == null
+ ? { tenantId: { $exists: false } }
+ : { tenantId: input.tenantId }),
+ 'envelope.target.bindingId': input.bindingId,
+ actorDetachedAction: { $exists: true },
+ })
+ .select('+actorDetachedAction')
+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();
+ return delivery?.actorDetachedAction ?? null;
+ }
+
async function getAgentEventActorReceipt(
input: GetAgentEventActorReceiptInput,
): Promise<AgentEventActorReceipt | null> {
@@ -1853,14 +2400,14 @@ export function createAgentTriggerDeliveryMethods(
]),
Delivery().countDocuments({ actorReceipt: { $exists: true }, expiresAt: { $lte: now } }),
Delivery().countDocuments({
- status: 'pending',
+ status: { $in: ['pending', 'capability_pending'] },
attempts: { $gt: 0 },
'envelope.target.bindingId': { $exists: true },
'handling.status': 'started',
actorReceipt: { $exists: false },
}),
Delivery().countDocuments({
- status: 'dead',
+ status: { $in: ['dead', 'capability_dead'] },
'envelope.target.bindingId': { $exists: true },
'handling.status': 'started',
actorReceipt: { $exists: false },
@@ -1887,8 +2434,8 @@ export function createAgentTriggerDeliveryMethods(
},
): Promise<boolean> {
const error = normalizeFailure(input.error);
- const result = await Delivery().updateOne(fence(input), {
- $set: { status: 'pending', availableAt: input.availableAt, lastError: error },
+ const retryUpdate = (status: 'pending' | 'capability_pending') => ({
+ $set: { status, availableAt: input.availableAt, lastError: error },
$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, settledAt: 1, expiresAt: 1 },
$push: {
history: {
@@ -1905,6 +2452,14 @@ export function createAgentTriggerDeliveryMethods(
},
},
});
+ const capabilityResult = await Delivery().updateOne(
+ capabilityFence(input),
+ retryUpdate('capability_pending'),
+ );
+ if (capabilityResult.modifiedCount === 1) {
+ return true;
+ }
+ const result = await Delivery().updateOne(fence(input), retryUpdate('pending'));
return result.modifiedCount === 1;
}
@@ -1916,29 +2471,33 @@ export function createAgentTriggerDeliveryMethods(
},
): Promise<boolean> {
const error = normalizeFailure(input.error);
- const dead = await Delivery()
- .findOneAndUpdate(
- fence(input),
- {
- $set: { status: 'dead', settledAt: input.settledAt, lastError: error },
- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },
- $push: {
- history: {
- $each: [
- {
- attempt: input.attempt,
- outcome: 'dead',
- at: input.settledAt,
- workerId: input.workerId,
- error,
- },
- ],
- $slice: -HISTORY_LIMIT,
+ const deadUpdate = (status: 'dead' | 'capability_dead') => ({
+ $set: { status, settledAt: input.settledAt, lastError: error },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },
+ $push: {
+ history: {
+ $each: [
+ {
+ attempt: input.attempt,
+ outcome: 'dead' as const,
+ at: input.settledAt,
+ workerId: input.workerId,
+ error,
},
- },
+ ],
+ $slice: -HISTORY_LIMIT,
},
- { new: true },
- )
+ },
+ });
+ const capabilityDead = await Delivery().updateOne(
+ capabilityFence(input),
+ deadUpdate('capability_dead'),
+ );
+ if (capabilityDead.modifiedCount === 1) {
+ return true;
+ }
+ const dead = await Delivery()
+ .findOneAndUpdate(fence(input), deadUpdate('dead'), { new: true })
.select('_id orderingKey batchMemberIds batchMembersSettledAt requeueCount')
.lean<
Pick<
@@ -2039,7 +2598,7 @@ export function createAgentTriggerDeliveryMethods(
}
const boundedLimit = Math.min(limit, MAX_DEAD_LETTER_LIMIT);
const deliveries = await Delivery()
- .find({ status: 'dead', batchRootId: { $exists: false } })
+ .find({ status: { $in: ['dead', 'capability_dead'] }, batchRootId: { $exists: false } })
.sort({ updatedAt: -1, _id: -1 })
.limit(boundedLimit)
.lean<IAgentTriggerDelivery[]>();
@@ -2053,7 +2612,7 @@ export function createAgentTriggerDeliveryMethods(
const candidate = await Delivery()
.findOne({
_id: id,
- status: 'dead',
+ status: { $in: ['dead', 'capability_dead'] },
batchRootId: { $exists: false },
actorReceipt: { $exists: false },
actorActionAdmittedAt: { $exists: false },
@@ -2071,7 +2630,7 @@ export function createAgentTriggerDeliveryMethods(
.findOneAndUpdate(
{
_id: candidate._id,
- status: 'dead',
+ status: candidate.status,
batchRootId: { $exists: false },
actorReceipt: { $exists: false },
actorActionAdmittedAt: { $exists: false },
@@ -2080,7 +2639,7 @@ export function createAgentTriggerDeliveryMethods(
},
{
$set: {
- status: 'staging',
+ status: candidate.requiredWorkerCapability == null ? 'staging' : 'capability_staging',
laneSequence: 0,
attempts: 0,
availableAt,
@@ -2116,7 +2675,24 @@ export function createAgentTriggerDeliveryMethods(
user: string | Types.ObjectId,
now: Date,
): Promise<number> {
- return Delivery().countDocuments({ user, status: 'leased', leaseUntil: { $gt: now } });
+ return Delivery().countDocuments({
+ user,
+ $or: [
+ {
+ actorActionAdmittedAt: { $exists: true },
+ actorReceipt: { $exists: false },
+ },
+ {
+ status: { $in: ['leased', 'capability_leased'] },
+ leaseUntil: { $gt: now },
+ },
+ {
+ 'actorDetachedAction.status': {
+ $in: ['reserved', 'running', 'launch_indeterminate'],
+ },
+ },
+ ],
+ });
}
function requireValidFence(fenceStartedAt: Date): void {
@@ -2149,6 +2725,10 @@ export function createAgentTriggerDeliveryMethods(
},
{ upsert: true },
);
+ await Delivery().updateMany(
+ { user },
+ { $set: { actorActionAdmissionClosedAt: fenceStartedAt } },
+ );
}
/** Disarms only the pre-commit deletion attempt that owns this marker. */
@@ -2164,8 +2744,12 @@ export function createAgentTriggerDeliveryMethods(
if (ownsLiveFence == null) {
return false;
}
- const result = await UserPurge().deleteOne({ _id: user, fenceStartedAt });
- return result.deletedCount === 1;
+ await Delivery().updateMany(
+ { user, actorActionAdmissionClosedAt: fenceStartedAt },
+ { $unset: { actorActionAdmissionClosedAt: 1 } },
+ );
+ await UserPurge().deleteOne({ _id: user, fenceStartedAt });
+ return true;
}
/** Recovers cleanup markers whose users are gone; active-user markers are never destructive. */
@@ -2195,6 +2779,10 @@ export function createAgentTriggerDeliveryMethods(
);
continue;
}
+ await Delivery().updateMany(
+ { user: marker._id, actorActionAdmissionClosedAt: marker.fenceStartedAt },
+ { $unset: { actorActionAdmissionClosedAt: 1 } },
+ );
await UserPurge().deleteOne({ _id: marker._id, fenceStartedAt: marker.fenceStartedAt });
continue;
}
@@ -2231,6 +2819,11 @@ export function createAgentTriggerDeliveryMethods(
releaseAgentEventActorAction,
getAgentEventActorActionAdmission,
hasAgentEventActorActionAdmission,
+ reserveAgentEventActorDetachedAction,
+ markAgentEventActorDetachedActionRunning,
+ markAgentEventActorDetachedActionLaunchIndeterminate,
+ settleAgentEventActorDetachedAction,
+ getAgentEventActorDetachedAction,
settleAgentEventActorReceipt,
getAgentEventActorReceipt,
backfillAgentEventActorReceipt,
diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts
index 186fa8fa5b..6ce9caf8d4 100644
--- a/packages/data-schemas/src/schema/convo.ts
+++ b/packages/data-schemas/src/schema/convo.ts
@@ -234,6 +234,20 @@ const convoSchema: Schema<IConversation> = new Schema(
agentEventActorSuspension: {
type: {
suspension: { type: Schema.Types.Mixed, required: true },
+ kind: {
+ type: String,
+ enum: ['human_decision', 'internal_completion'],
+ default: 'human_decision',
+ },
+ appliedAction: {
+ type: {
+ toolName: { type: String, required: true },
+ toolCallId: { type: String, default: undefined },
+ },
+ _id: false,
+ default: undefined,
+ },
+ handlingGenerationCreatedAt: { type: Number, min: 0, default: undefined },
actionId: { type: String, required: true },
jobCreatedAt: { type: Number, required: true },
status: { type: String, enum: ['pending', 'claimed', 'closed'], required: true },
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
index 7d23845fb7..c3dc166c40 100644
--- a/packages/data-schemas/src/schema/triggerDelivery.ts
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -85,6 +85,33 @@ const actorReceiptSchema = new Schema(
{ _id: false },
);
+const actorDetachedActionSchema = new Schema(
+ {
+ version: { type: Number, enum: [1], required: true },
+ invocationId: { type: String, required: true, maxlength: 128 },
+ expectedToolName: { type: String, required: true, maxlength: 256 },
+ toolName: { type: String, required: true, maxlength: 256 },
+ toolCallId: { type: String, required: true, maxlength: 256 },
+ turnId: { type: String, required: true, maxlength: 512 },
+ taskId: { type: String, required: true, maxlength: 128 },
+ idempotencyKey: { type: String, required: true, minlength: 64, maxlength: 64 },
+ launchAttempt: { type: Number, min: 0, max: 15, required: true },
+ status: {
+ type: String,
+ enum: ['reserved', 'running', 'launch_indeterminate', 'succeeded', 'failed', 'cancelled'],
+ required: true,
+ },
+ reservedAt: { type: Date, required: true },
+ observedAt: { type: Date, required: true },
+ recoveryAfter: { type: Date, required: true },
+ launchedAt: { type: Date },
+ settledAt: { type: Date },
+ result: { type: String, maxlength: 32_768 },
+ error: { type: String, maxlength: 2_048 },
+ },
+ { _id: false },
+);
+
const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
{
deliveryKey: { type: String, required: true, maxlength: 128 },
@@ -97,10 +124,22 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
tenantId: { type: String, index: true },
status: {
type: String,
- enum: ['staging', 'batched', 'pending', 'leased', 'succeeded', 'dead'],
+ enum: [
+ 'staging',
+ 'capability_staging',
+ 'batched',
+ 'pending',
+ 'capability_pending',
+ 'leased',
+ 'capability_leased',
+ 'succeeded',
+ 'capability_dead',
+ 'dead',
+ ],
required: true,
default: 'pending',
},
+ requiredWorkerCapability: { type: String, maxlength: 128 },
attempts: { type: Number, required: true, default: 0, min: 0 },
availableAt: { type: Date, required: true },
envelopeBytes: { type: Number, min: 0 },
@@ -119,8 +158,15 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
awaitTerminalHandling: { type: Boolean },
handling: { type: handlingSchema },
actorReceipt: { type: actorReceiptSchema, select: false },
+ actorDetachedAction: { type: actorDetachedActionSchema, select: false },
+ actorDetachedActionHistory: {
+ type: [actorDetachedActionSchema],
+ default: undefined,
+ select: false,
+ },
actorActionAdmittedAt: { type: Date, select: false },
actorActionAdmissionId: { type: String, maxlength: 64, select: false },
+ actorActionAdmissionClosedAt: { type: Date, select: false },
leaseBy: { type: String },
leaseUntil: { type: Date },
claimToken: { type: String },
@@ -155,6 +201,7 @@ triggerDeliverySchema.index(
);
triggerDeliverySchema.index({ status: 1, updatedAt: -1 });
triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });
+triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });
// Only successful rows receive expiresAt. Dead letters remain available until
diff --git a/packages/data-schemas/src/schema/triggerLaneSequence.ts b/packages/data-schemas/src/schema/triggerLaneSequence.ts
index 5a6bbb1369..bcb92fd8bc 100644
--- a/packages/data-schemas/src/schema/triggerLaneSequence.ts
+++ b/packages/data-schemas/src/schema/triggerLaneSequence.ts
@@ -9,6 +9,7 @@ const triggerLaneSequenceSchema: Schema<IAgentTriggerLaneSequenceDocument> = new
tenantId: { type: String, index: true },
tailDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },
publisherDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },
+ publisherRequeueCount: { type: Number, min: 0 },
publisherStartedAt: { type: Date },
cleanupRequestedAt: { type: Date, index: true },
},
diff --git a/packages/data-schemas/src/types/convo.ts b/packages/data-schemas/src/types/convo.ts
index 0b8a94a009..bfe8601078 100644
--- a/packages/data-schemas/src/types/convo.ts
+++ b/packages/data-schemas/src/types/convo.ts
@@ -146,6 +146,15 @@ export interface IAgentEventActorSuspensionEvidence {
*/
export interface IAgentEventActorSuspension {
suspension: IAgentEventActorSuspensionEvidence;
+ /** Host-side reason for suspension. Missing legacy values are human decisions. */
+ kind?: 'human_decision' | 'internal_completion';
+ /** Expected-action evidence already applied before a later re-pause. */
+ appliedAction?: {
+ toolName: string;
+ toolCallId?: string;
+ };
+ /** Original delivery-handling generation retained across resumed generations. */
+ handlingGenerationCreatedAt?: number;
actionId: string;
jobCreatedAt: number;
status: 'pending' | 'claimed' | 'closed';
diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts
index 450582629b..671e6c63c7 100644
--- a/packages/data-schemas/src/types/triggerDelivery.ts
+++ b/packages/data-schemas/src/types/triggerDelivery.ts
@@ -3,11 +3,16 @@ import type { IAgentEventActorReconciliation } from './convo';
export type AgentTriggerDeliveryStatus =
| 'staging'
+ | 'capability_staging'
| 'batched'
| 'pending'
+ | 'capability_pending'
| 'leased'
+ | 'capability_leased'
| 'succeeded'
+ | 'capability_dead'
| 'dead';
+export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1';
export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';
export interface AgentTriggerHandlingState {
@@ -36,6 +41,30 @@ export interface AgentEventActorReceipt {
settledAt: Date;
}
+/** Private launch authority for one delivery-owned detached expected action. */
+export interface AgentEventActorDetachedAction {
+ version: 1;
+ invocationId: string;
+ expectedToolName: string;
+ toolName: string;
+ toolCallId: string;
+ /** Stable graph-turn identity, independent of retry allocation. */
+ turnId: string;
+ taskId: string;
+ idempotencyKey: string;
+ launchAttempt: number;
+ status: 'reserved' | 'running' | 'launch_indeterminate' | 'succeeded' | 'failed' | 'cancelled';
+ reservedAt: Date;
+ observedAt: Date;
+ /** A recovery fence, not relaunch authority. Expiry only permits the exact
+ * launch to be marked indeterminate while late terminal proof remains valid. */
+ recoveryAfter: Date;
+ launchedAt?: Date;
+ settledAt?: Date;
+ result?: string;
+ error?: string;
+}
+
export interface AgentTriggerDeliveryFailure {
code: string;
message: string;
@@ -64,6 +93,8 @@ export interface IAgentTriggerDelivery {
user: Types.ObjectId;
tenantId?: string;
status: AgentTriggerDeliveryStatus;
+ /** Keeps a delivery invisible to pre-capability workers during rolling deploys. */
+ requiredWorkerCapability?: string;
attempts: number;
availableAt: Date;
envelopeBytes?: number;
@@ -81,11 +112,17 @@ export interface IAgentTriggerDelivery {
awaitTerminalHandling?: boolean;
handling?: AgentTriggerHandlingState;
actorReceipt?: AgentEventActorReceipt;
+ /** Durable launch identity; excluded from ordinary delivery reads. */
+ actorDetachedAction?: AgentEventActorDetachedAction;
+ /** Bounded audit trail for terminal attempts replaced by an explicit retry. */
+ actorDetachedActionHistory?: AgentEventActorDetachedAction[];
/** Delivery-owned serialization point acquired exactly once before an event
* actor may invoke an external action. */
actorActionAdmittedAt?: Date;
/** Attempt identity that fences admission takeover and release. */
actorActionAdmissionId?: string;
+ /** Account-deletion fence that atomically closes action admission on this delivery. */
+ actorActionAdmissionClosedAt?: Date;
leaseBy?: string;
leaseUntil?: Date;
claimToken?: string;
@@ -136,6 +173,8 @@ export interface IAgentTriggerLaneSequence {
tailDeliveryId?: Types.ObjectId;
/** Delivery currently owning the serialized sequence/publication step. */
publisherDeliveryId?: Types.ObjectId;
+ /** Requeue generation captured when this publisher reservation was acquired. */
+ publisherRequeueCount?: number;
publisherStartedAt?: Date;
cleanupRequestedAt?: Date;
createdAt?: Date;
@@ -163,7 +202,7 @@ export interface AgentTriggerDeliveryClaim extends AgentTriggerDeliveryRecord {
claimToken: string;
leaseBy: string;
leaseUntil: Date;
- status: 'leased';
+ status: 'leased' | 'capability_leased';
}
export interface AgentTriggerOrderingBlock {