Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🪂 feat: Make Event Actor Detached Actions Durable (#15307)
Danny Avila487984193a8d
diff --git a/.env.example b/.env.exampleindex 9b9cea96ed..1c21df2889 100644--- a/.env.example+++ b/.env.example@@ -197,6 +197,10 @@ LOG_TO_FILE=true# Set to true to enable agent debug loggingAGENT_DEBUG_LOGGING=false+# Rolling-deployment fence for durable detached Event Actor completions. Requires Redis generation+# streams. Leave disabled while deploying consumers, then enable after every replica is updated.+# AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED=false+# Enable memory diagnostics (logs heap/RSS snapshots every 60s, auto-enabled with --inspect)# MEM_DIAG=truediff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.jsindex f0bd8e2d03..f8b30502a2 100644--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js@@ -88,6 +88,7 @@ jest.mock('@librechat/api', () => ({conversationId,parentMessageId,}),+ parseAgentEventActorDetachedCompletion: jest.fn(() => undefined),}));jest.mock('~/server/cleanup', () => ({diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex 2b37fd53c8..d93eb82f68 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -8,6 +8,7 @@ const mockLogger = {};const mockGenerationJobManager = {+ isRedis: false,createJob: jest.fn(),getJob: jest.fn(),emitError: jest.fn(),@@ -22,6 +23,7 @@ const mockGenerationJobManager = {failPausePersistence: jest.fn(),getResumeState: jest.fn(),updateMetadata: jest.fn(),+ persistAgentEventDetachedTerminalEvidence: jest.fn(),claimGeneration: jest.fn(),resumeClaimedGeneration: jest.fn(),takeoverGeneration: jest.fn(),@@ -73,6 +75,8 @@ const mockRecordScheduleOutcome = jest.fn();const mockIsScheduleLive = jest.fn();const mockDeleteAgentCheckpoint = jest.fn();const mockExecuteAgentEventActor = jest.fn();+const mockResumeAgentEventActor = jest.fn();+const mockCreateAgentEventActorDetachedActionLifecycle = jest.fn(() => undefined);const mockFindAgentEventAppliedAction = jest.fn();const mockResolveAgentTurnExecutionPlan = jest.fn((input) => {let origin = 'user';@@ -85,7 +89,6 @@ const mockResolveAgentTurnExecutionPlan = jest.fn((input) => {input.event?.binding != null &&input.event?.expectedAction != null &&input.checkpointerType !== 'memory' &&- !input.expectedActionMayDetach &&(!input.canPause || input.durableEventActorSuspensions);let strategy = 'history';if (input.isNewConversation) {@@ -131,6 +134,9 @@ const mockAdmitAgentEventActorAction = jest.fn();const mockReleaseAgentEventActorAction = jest.fn();const mockHasAgentEventActorActionAdmission = jest.fn();const mockGetAgentEventActorReceipt = jest.fn();+const mockGetAgentEventActorDetachedAction = jest.fn();+const mockClaimAgentEventActorSuspension = jest.fn();+const mockSettleAgentEventActorSuspension = jest.fn();const mockStartupTelemetry = {mark: jest.fn(),setStreamId: jest.fn(),@@ -258,6 +264,13 @@ jest.mock('@librechat/api', () => ({},deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),executeAgentEventActor: (...args) => mockExecuteAgentEventActor(...args),+ resumeAgentEventActor: (...args) => mockResumeAgentEventActor(...args),+ createAgentEventActorDetachedActionLifecycle: (...args) =>+ mockCreateAgentEventActorDetachedActionLifecycle(...args),+ isAgentEventActorDetachedActionProducerEnabled: () => true,+ parseAgentEventActorDetachedCompletion: (value) => (value?.version === 1 ? value : undefined),+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE: 'librechat-event-actor',+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE: 'librechat.event_actor.detached_completion',findAgentEventAppliedAction: (...args) => mockFindAgentEventAppliedAction(...args),createAgentEventActionRecorder: (...args) => mockCreateAgentEventActionRecorder(...args),resolveAgentTurnExecutionPlan: (...args) => mockResolveAgentTurnExecutionPlan(...args),@@ -307,6 +320,9 @@ jest.mock('~/models', () => ({releaseAgentEventActorAction: (...args) => mockReleaseAgentEventActorAction(...args),hasAgentEventActorActionAdmission: (...args) => mockHasAgentEventActorActionAdmission(...args),getAgentEventActorReceipt: (...args) => mockGetAgentEventActorReceipt(...args),+ getAgentEventActorDetachedAction: (...args) => mockGetAgentEventActorDetachedAction(...args),+ claimAgentEventActorSuspension: (...args) => mockClaimAgentEventActorSuspension(...args),+ settleAgentEventActorSuspension: (...args) => mockSettleAgentEventActorSuspension(...args),isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args),}));@@ -378,6 +394,8 @@ describe('ResumableAgentController resume metadata', () => {mockGenerationJobManager.getResumeState.mockResolvedValue(null);mockGenerationJobManager.getJob.mockResolvedValue(undefined);mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);+ mockGenerationJobManager.isRedis = false;+ mockGenerationJobManager.persistAgentEventDetachedTerminalEvidence.mockResolvedValue(true);mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);mockGenerationJobManager.emitDone.mockResolvedValue(undefined);mockGenerationJobManager.emitError.mockResolvedValue(undefined);@@ -433,6 +451,9 @@ describe('ResumableAgentController resume metadata', () => {mockReleaseAgentEventActorAction.mockResolvedValue(true);mockHasAgentEventActorActionAdmission.mockResolvedValue(false);mockGetAgentEventActorReceipt.mockResolvedValue(null);+ mockGetAgentEventActorDetachedAction.mockResolvedValue(null);+ mockClaimAgentEventActorSuspension.mockResolvedValue({ status: 'claimed' });+ mockSettleAgentEventActorSuspension.mockResolvedValue({ status: 'settled' });});it.each([@@ -4110,7 +4131,6 @@ describe('ResumableAgentController resume metadata', () => {conversationId: 'child-conversation',isNewConversation: false,canPause: false,- expectedActionMayDetach: false,}),);expect(mockExecuteAgentEventActor).toHaveBeenCalledWith(@@ -4120,7 +4140,7 @@ describe('ResumableAgentController resume metadata', () => {event,expectedAction: { toolName: 'submit_\u200Fmove', argumentSubset: { expectedPly: 8 } },}),- {+ expect.objectContaining({getSnapshot: expect.any(Function),commitState: expect.any(Function),recordReconciliation: expect.any(Function),@@ -4130,7 +4150,7 @@ describe('ResumableAgentController resume metadata', () => {releaseAction: expect.any(Function),hasActionAdmission: expect.any(Function),getReceipt: expect.any(Function),- },+ }),);expect(mockExecuteAgentEventActor.mock.calls[0][0]).not.toHaveProperty('tenantId');expect(req._agentEventTriggerProjection).toEqual({@@ -4231,6 +4251,158 @@ describe('ResumableAgentController resume metadata', () => {expect(mockFindAgentEventAppliedAction).not.toHaveBeenCalled();});+ it('resumes the original actor suspension from exact detached terminal evidence', async () => {+ mockGenerationJobManager.isRedis = true;+ mockGenerationJobManager.createJob.mockResolvedValue({+ createdAt: 2000,+ metadata: {+ checkpointNamespace: '2000',+ providerExecutionId: 'provider-segment-2',+ providerDrained: true,+ },+ readyPromise: Promise.resolve(),+ abortController: new AbortController(),+ emitter: { on: jest.fn() },+ });+ mockGenerationJobManager.claimGeneration.mockResolvedValue(+ wonGenerationClaim({ streamId: 'child-conversation', conversationId: 'child-conversation' }),+ );+ mockGetConvo.mockResolvedValue({+ conversationId: 'parent-conversation',+ agent_id: 'parent-agent',+ tenantId: 'tenant-1',+ });+ const suspension = {+ version: 1,+ suspensionId: 'suspension-detached-1',+ attempt: 0,+ invocation: { invocationId: 'original-delivery-1' },+ };+ mockGetAgentEventActorSnapshot.mockResolvedValue({+ state: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension,+ actionId: 'task-detached-1',+ status: 'pending',+ },+ });+ mockGetAgentEventActorDetachedAction.mockResolvedValue({+ invocationId: 'original-delivery-1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-detached-1',+ taskId: 'task-detached-1',+ idempotencyKey: 'a'.repeat(64),+ status: 'succeeded',+ result: 'move accepted',+ });+ const repaused = { kind: 'internal_completion', actionId: 'task-detached-2' };+ mockCreateAgentEventActorDetachedActionLifecycle.mockReturnValueOnce({+ readSuspension: () => repaused,+ });+ mockResumeAgentEventActor.mockImplementationOnce(async (input) => {+ expect(input.readSuspension()).toBe(repaused);+ throw new Error('stop after resume contract');+ });+ const client = { options: { agent: {} }, sendMessage: jest.fn() };+ const req = {+ user: { id: 'user-123', tenantId: 'tenant-1' },+ body: {+ text: 'Trusted detached completion.',+ clientRequestId: 'detached-resume-generation-1',+ conversationId: 'child-conversation',+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },+ agentEventDelivery: {+ deliveryKey: 'detached-resume-generation-1',+ target: { bindingId: 'binding-1' },+ expectedAction: { toolName: 'submit_move' },+ event: {+ id: 'task-detached-1',+ type: 'librechat.event_actor.detached_completion',+ occurredAt: Date.now(),+ source: { id: 'librechat-event-actor', type: 'internal' },+ },+ internalCompletion: {+ version: 1,+ invocationId: 'original-delivery-1',+ generationCreatedAt: 1000,+ wakeGenerationCreatedAt: 2000,+ taskId: 'task-detached-1',+ idempotencyKey: 'a'.repeat(64),+ },+ },+ },+ config: { endpoints: { agents: {} } },+ _isAgentTrigger: true,+ _agentEventBindingId: 'binding-1',+ _agentEventBindingParentConversationId: 'parent-conversation',+ _agentEventBindingParentAgentId: 'parent-agent',+ _agentEventBindingTenantId: 'tenant-1',+ _agentEventBindingRetention: { expiredAt: new Date(Date.now() + 60_000) },+ };++ await AgentController(+ req,+ createResumableResponse(),+ jest.fn(),+ jest.fn().mockResolvedValue({ client }),+ null,+ );+ await nextTick();++ expect(mockExecuteAgentEventActor).not.toHaveBeenCalled();+ expect(mockResumeAgentEventActor).toHaveBeenCalledWith(+ expect.objectContaining({+ conversationId: 'child-conversation',+ bindingId: 'binding-1',+ suspension,+ resumeAttemptId: 'detached-resume-generation-1',+ resumeValue: expect.objectContaining({+ taskId: 'task-detached-1',+ status: 'succeeded',+ result: 'move accepted',+ }),+ }),+ expect.objectContaining({+ claimSuspension: expect.any(Function),+ settleSuspension: expect.any(Function),+ }),+ );+ expect(mockGetAgentEventActorDetachedAction).toHaveBeenCalledWith(+ expect.objectContaining({+ deliveryKey: 'original-delivery-1',+ generationCreatedAt: 1000,+ }),+ );+ expect(mockCreateAgentEventActorDetachedActionLifecycle).toHaveBeenCalledWith(+ expect.objectContaining({+ generationCreatedAt: 1000,+ turnCreatedAt: 2000,+ }),+ expect.any(Object),+ );+ const lifecycleDependencies =+ mockCreateAgentEventActorDetachedActionLifecycle.mock.calls.at(-1)[1];+ expect(lifecycleDependencies.producerEnabled()).toBe(true);+ mockGenerationJobManager.isRedis = false;+ expect(lifecycleDependencies.producerEnabled()).toBe(false);+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(+ 'child-conversation',+ 'user-123',+ 'child-conversation',+ expect.objectContaining({+ initialMetadata: expect.objectContaining({+ agentEventDeliveryKey: 'detached-resume-generation-1',+ agentEventInvocationKey: 'original-delivery-1',+ agentEventInvocationGenerationCreatedAt: 1000,+ agentEventDetachedActionProducerRequired: true,+ }),+ }),+ );+ });+it('records reconciliation when persistence fails after an event action commits', async () => {mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim({ streamId: 'child-conversation', conversationId: 'child-conversation' }),@@ -4530,9 +4702,7 @@ describe('ResumableAgentController resume metadata', () => {}),};const shouldCheckpoint =- _label !== 'memory-checkpointer' &&- _label !== 'background-capable expected action' &&- _label !== 'pre-cutover pause-capable fleet';+ _label !== 'memory-checkpointer' && _label !== 'pre-cutover pause-capable fleet';if (shouldCheckpoint) {mockExecuteAgentEventActor.mockImplementationOnce(async (input) => {await input.invoke({@@ -4671,7 +4841,7 @@ describe('ResumableAgentController resume metadata', () => {},},config: {- endpoints: { agents: {} },+ endpoints: { agents: { checkpointer: { type: 'memory' } } },},_isAgentTrigger: true,_agentEventBindingId: 'binding-1',@@ -4746,7 +4916,7 @@ describe('ResumableAgentController resume metadata', () => {},},config: {- endpoints: { agents: {} },+ endpoints: { agents: { checkpointer: { type: 'memory' } } },},_isAgentTrigger: true,_agentEventBindingId: 'binding-1',@@ -4817,7 +4987,7 @@ describe('ResumableAgentController resume metadata', () => {},},config: {- endpoints: { agents: {} },+ endpoints: { agents: { checkpointer: { type: 'memory' } } },},_isAgentTrigger: true,_agentEventBindingId: 'binding-1',@@ -4899,7 +5069,7 @@ describe('ResumableAgentController resume metadata', () => {},},config: {- endpoints: { agents: {} },+ endpoints: { agents: { checkpointer: { type: 'memory' } } },},_isAgentTrigger: true,_agentEventBindingId: 'binding-1',diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.jsindex 8b73032b25..3d8646ccec 100644--- a/api/server/controllers/agents/__tests__/resume.spec.js+++ b/api/server/controllers/agents/__tests__/resume.spec.js@@ -50,6 +50,7 @@ const mockGenerationJobManager = {getJob: jest.fn(),getJobStore: jest.fn(() => mockJobStore),getResumeState: jest.fn(),+ updateMetadata: jest.fn(),setContentParts: jest.fn(),/** Resume moves ownership and rebuilds armed interrupts from the durable queue. */rearmQueuedPreempts: jest.fn().mockResolvedValue(0),@@ -110,6 +111,12 @@ const mockClaimAgentEventActorSuspension = jest.fn();const mockSettleAgentEventActorSuspension = jest.fn();const mockRecordAgentEventActorReconciliation = jest.fn();const mockResumeAgentEventActor = jest.fn();+const mockCreateAgentEventActorDetachedActionLifecycle = jest.fn();+const mockIsAgentEventActorDetachedActionProducerEnabled = jest.fn();+const mockGetAgentTriggerDelivery = jest.fn();+const mockReserveAgentEventActorDetachedAction = jest.fn();+const mockMarkAgentEventActorDetachedActionRunning = jest.fn();+const mockSettleAgentEventActorDetachedAction = jest.fn();jest.mock('@librechat/data-schemas', () => ({...jest.requireActual('@librechat/data-schemas'),@@ -132,6 +139,10 @@ jest.mock('@librechat/api', () => ({getAgentCheckpointer: (...args) => mockGetAgentCheckpointer(...args),checkAccess: (...args) => mockCheckAccess(...args),resumeAgentEventActor: (...args) => mockResumeAgentEventActor(...args),+ createAgentEventActorDetachedActionLifecycle: (...args) =>+ mockCreateAgentEventActorDetachedActionLifecycle(...args),+ isAgentEventActorDetachedActionProducerEnabled: () =>+ mockIsAgentEventActorDetachedActionProducerEnabled(),}));jest.mock('~/models', () => ({@@ -143,6 +154,7 @@ jest.mock('~/models', () => ({getActions: (...args) => mockGetActions(...args),getUserMemories: (...args) => mockGetUserMemories(...args),getRoleByName: (...args) => mockGetRoleByName(...args),+ getAgentTriggerDelivery: (...args) => mockGetAgentTriggerDelivery(...args),isSubagentOwnerAdmissible: (...args) => mockIsSubagentOwnerAdmissible(...args),completeAgentEventActorLegacyTurn: (...args) => mockCompleteAgentEventActorLegacyTurn(...args),getAgentEventActorSnapshot: (...args) => mockGetAgentEventActorSnapshot(...args),@@ -152,6 +164,12 @@ jest.mock('~/models', () => ({settleAgentEventActorSuspension: (...args) => mockSettleAgentEventActorSuspension(...args),recordAgentEventActorReconciliation: (...args) =>mockRecordAgentEventActorReconciliation(...args),+ reserveAgentEventActorDetachedAction: (...args) =>+ mockReserveAgentEventActorDetachedAction(...args),+ markAgentEventActorDetachedActionRunning: (...args) =>+ mockMarkAgentEventActorDetachedActionRunning(...args),+ settleAgentEventActorDetachedAction: (...args) =>+ mockSettleAgentEventActorDetachedAction(...args),}));jest.mock('~/server/services/Endpoints/agents/eventChildLease', () => ({@@ -325,6 +343,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {});mockJobStore.updateJob.mockResolvedValue(undefined);mockGenerationJobManager.getResumeState.mockResolvedValue({ aggregatedContent: [] });+ mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);mockGenerationJobManager.emitDone.mockResolvedValue(undefined);mockGenerationJobManager.emitError.mockResolvedValue(undefined);mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);@@ -352,6 +371,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);+ mockGenerationJobManager.isRedis = true;+ mockGenerationJobManager.persistAgentEventDetachedTerminalEvidence = jest+ .fn()+ .mockResolvedValue(true);+ mockGenerationJobManager.retryTerminalHostAction = jest.fn().mockResolvedValue(undefined);mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);@@ -375,6 +399,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {mockClaimAgentEventActorSuspension.mockResolvedValue({ status: 'claimed' });mockSettleAgentEventActorSuspension.mockResolvedValue({ status: 'settled' });mockRecordAgentEventActorReconciliation.mockResolvedValue(true);+ mockIsAgentEventActorDetachedActionProducerEnabled.mockReturnValue(true);+ mockGetAgentTriggerDelivery.mockResolvedValue(undefined);+ mockCreateAgentEventActorDetachedActionLifecycle.mockReturnValue(undefined);endpointAgent = {_id: 'mongo-agent-abc',id: AGENT_ID,@@ -501,6 +528,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {const pausedJob = makeToolApprovalJob({metadata: {idempotencyClientRequestId: 'trigger_event_delivery',+ agentEventDeliveryKey: 'completion-delivery-1',+ agentEventInvocationKey: 'trigger_event_delivery',agentEventExpectedAction: expectedAction,agentEventSuspension: {version: 1,@@ -513,6 +542,18 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {{ tool_call_id: 'tc1', allowed_decisions: ['respond'] },];mockGenerationJobManager.getJob.mockResolvedValue(pausedJob);+ const detachedSuspension = {+ kind: 'internal_completion',+ actionId: 'detached-action-next',+ };+ const repausedSuspension = {+ ...suspension,+ suspensionId: 'suspension-detached-next',+ attempt: 1,+ };+ mockCreateAgentEventActorDetachedActionLifecycle.mockReturnValue({+ readSuspension: () => detachedSuspension,+ });let projectedProviderExecutionId;mockGenerationJobManager.approvals.resolve.mockImplementation(async (_streamId, _actionId, resumePatch) => {@@ -535,12 +576,15 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {reconciliations: [],suspension: {suspension,+ kind: 'internal_completion',+ handlingGenerationCreatedAt: 500,actionId: ACTION_ID,jobCreatedAt: 1000,status: 'pending',},});const resumedClient = makeClient({+ publishStagedApproval: jest.fn(),contentParts: [makeToolCallContent({ output: 'human supplied output' })],run: {getRunSteps: () => [@@ -562,8 +606,14 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {],},});- mockInitializeClient.mockResolvedValue({ client: resumedClient, userMCPAuthMap: {} });+ mockInitializeClient.mockImplementation(async ({ req }) => {+ expect(req._agentEventDetachedActionLifecycle).toBe(+ mockCreateAgentEventActorDetachedActionLifecycle.mock.results[0].value,+ );+ return { client: resumedClient, userMCPAuthMap: {} };+ });mockResumeAgentEventActor.mockImplementation(async (input, dependencies) => {+ expect(input.readSuspension()).toBe(detachedSuspension);await dependencies.claimSuspension({user: USER_ID,tenantId: TENANT_ID,@@ -583,10 +633,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {continuation: 'warm',signal: input.signal,});- expect(input.readAppliedAction()).toBeUndefined();return {value,- execution: { status: 'completed_no_action' },+ execution: { status: 'suspended', suspension: repausedSuspension },};});@@ -610,40 +659,60 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0],);expect(mockGenerationJobManager.getJob).toHaveBeenCalledTimes(2);+ expect(mockGetAgentTriggerDelivery).not.toHaveBeenCalled();+ expect(mockCreateAgentEventActorDetachedActionLifecycle).toHaveBeenCalledWith(+ expect.objectContaining({+ invocationId: 'trigger_event_delivery',+ generationCreatedAt: 500,+ turnCreatedAt: 1000,+ expectedAction,+ }),+ expect.objectContaining({+ reserveAgentEventActorDetachedAction: expect.any(Function),+ markAgentEventActorDetachedActionRunning: expect.any(Function),+ settleAgentEventActorDetachedAction: expect.any(Function),+ }),+ );+ expect(+ mockCreateAgentEventActorDetachedActionLifecycle.mock.invocationCallOrder[0],+ ).toBeLessThan(mockInitializeClient.mock.invocationCallOrder[0]);+ const lifecycleDependencies =+ mockCreateAgentEventActorDetachedActionLifecycle.mock.calls[0][1];+ expect(lifecycleDependencies.producerEnabled()).toBe(true);+ mockGenerationJobManager.isRedis = false;+ expect(lifecycleDependencies.producerEnabled()).toBe(false);expect(resumedClient.resumeCompletion).toHaveBeenCalledTimes(1);+ expect(mockGenerationJobManager.updateMetadata).toHaveBeenCalledWith(+ CONVO_ID,+ {+ agentEventSuspension: {+ version: repausedSuspension.version,+ suspensionId: repausedSuspension.suspensionId,+ attempt: repausedSuspension.attempt,+ },+ },+ 1000,+ );+ expect(resumedClient.publishStagedApproval).not.toHaveBeenCalled();expect(mockRecordAgentEventActorReconciliation).not.toHaveBeenCalled();});- it('does not record provider execution when client reconstruction fails before continuation', async () => {+ it('refuses a prior-head expected-action suspension before claiming it on an incapable replica', async () => {configureEventActorResume();+ requestStateOverrides._agentEventBindingId = 'binding-1';+ const expectedAction = { toolName: 'submit_move' };const suspension = {version: 1,- suspensionId: 'suspension-init-failure',+ suspensionId: 'suspension-producer-required',attempt: 0,issuedAt: Date.now(),expiresAt: Date.now() + 60_000,- invocation: {- invocationId: 'trigger_event_delivery',- continuation: 'warm',- base: { actorThreadId: CONVO_ID, generation: 0 },- fork: {- threadId: CONVO_ID,- checkpointNs: 'event-actor',- checkpointId: 'checkpoint-paused',- invocationId: 'trigger_event_delivery',- },- },- checkpoint: {- threadId: CONVO_ID,- checkpointNs: 'event-actor',- checkpointId: 'checkpoint-paused',- invocationId: 'trigger_event_delivery',- },+ invocation: { invocationId: 'trigger_event_delivery' },interrupt: {- id: 'interrupt-init-failure',+ id: 'interrupt-producer-required',payload: {type: 'tool_approval',- _librechatEventActor: { expectedAction: { toolName: 'lookup' } },+ _librechatEventActor: { expectedAction },},},suspensionDigest: 'signed-digest',@@ -652,6 +721,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {makeToolApprovalJob({metadata: {idempotencyClientRequestId: 'trigger_event_delivery',+ agentEventDeliveryKey: 'trigger_event_delivery',+ agentEventExpectedAction: expectedAction,agentEventSuspension: {version: 1,suspensionId: suspension.suspensionId,@@ -667,33 +738,119 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {reconciliations: [],suspension: {suspension,+ kind: 'human_decision',+ handlingGenerationCreatedAt: 1000,actionId: ACTION_ID,jobCreatedAt: 1000,status: 'pending',},});- mockInitializeClient.mockRejectedValue(new Error('client reconstruction failed'));- mockResumeAgentEventActor.mockImplementation(async (input) => {- expect(await input.claimProjection()).toBe(true);- return input.resume({- checkpointNamespace: 'event-actor',- checkpointId: 'checkpoint-paused',- invocationId: 'trigger_event_delivery',- continuation: 'warm',- signal: input.signal,- });- });+ mockIsAgentEventActorDetachedActionProducerEnabled.mockReturnValue(false);const res = await post(approveBody());- expect(res.status).toBe(200);- await settled;- await flush();- expect(mockInitializeClient).toHaveBeenCalledTimes(1);- expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();- expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();+ expect(res.status).toBe(503);+ expect(res.body).toMatchObject({+ code: 'EVENT_ACTOR_RESUME_CAPABILITY_UNAVAILABLE',+ });+ expect(res.headers['retry-after']).toBe('1');+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();+ expect(mockResumeAgentEventActor).not.toHaveBeenCalled();+ expect(mockInitializeClient).not.toHaveBeenCalled();+ expect(mockDecrementPendingRequest).toHaveBeenCalledTimes(1);+ expect(mockReleaseEventChildLease).toHaveBeenCalledTimes(1);});+ it.each([+ ['pre-feature expected-action', { toolName: 'lookup' }, undefined],+ ['actionless PR-era', undefined, 1000],+ ])(+ 'preserves a %s suspension when client reconstruction fails before continuation',+ async (_case, expectedAction, handlingGenerationCreatedAt) => {+ configureEventActorResume();+ const suspension = {+ version: 1,+ suspensionId: 'suspension-init-failure',+ attempt: 0,+ issuedAt: Date.now(),+ expiresAt: Date.now() + 60_000,+ invocation: {+ invocationId: 'trigger_event_delivery',+ continuation: 'warm',+ base: { actorThreadId: CONVO_ID, generation: 0 },+ fork: {+ threadId: CONVO_ID,+ checkpointNs: 'event-actor',+ checkpointId: 'checkpoint-paused',+ invocationId: 'trigger_event_delivery',+ },+ },+ checkpoint: {+ threadId: CONVO_ID,+ checkpointNs: 'event-actor',+ checkpointId: 'checkpoint-paused',+ invocationId: 'trigger_event_delivery',+ },+ interrupt: {+ id: 'interrupt-init-failure',+ payload: {+ type: 'tool_approval',+ _librechatEventActor: expectedAction == null ? {} : { expectedAction },+ },+ },+ suspensionDigest: 'signed-digest',+ };+ mockGenerationJobManager.getJob.mockResolvedValue(+ makeToolApprovalJob({+ metadata: {+ idempotencyClientRequestId: 'trigger_event_delivery',+ ...(expectedAction == null ? {} : { agentEventExpectedAction: expectedAction }),+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: suspension.attempt,+ },+ },+ }),+ );+ mockGetAgentEventActorSnapshot.mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ suspension,+ ...(handlingGenerationCreatedAt == null ? {} : { handlingGenerationCreatedAt }),+ actionId: ACTION_ID,+ jobCreatedAt: 1000,+ status: 'pending',+ },+ });+ mockIsAgentEventActorDetachedActionProducerEnabled.mockReturnValue(false);+ mockInitializeClient.mockRejectedValue(new Error('client reconstruction failed'));+ mockResumeAgentEventActor.mockImplementation(async (input) => {+ expect(await input.claimProjection()).toBe(true);+ return input.resume({+ checkpointNamespace: 'event-actor',+ checkpointId: 'checkpoint-paused',+ invocationId: 'trigger_event_delivery',+ continuation: 'warm',+ signal: input.signal,+ });+ });++ const res = await post(approveBody());+ expect(res.status).toBe(200);+ await settled;+ await flush();++ expect(mockInitializeClient).toHaveBeenCalledTimes(1);+ expect(mockCreateAgentEventActorDetachedActionLifecycle).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalled();+ },+ );+it('fails closed when a versioned job marker no longer matches canonical suspension', async () => {configureEventActorResume();mockGenerationJobManager.getJob.mockResolvedValue(diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex 87ddfbde76..7cc2613c5c 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -31,6 +31,12 @@ const {createMCPRuntimeRequestBody,isAgentEventRetentionActive,executeAgentEventActor,+ resumeAgentEventActor,+ createAgentEventActorDetachedActionLifecycle,+ isAgentEventActorDetachedActionProducerEnabled,+ parseAgentEventActorDetachedCompletion,+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,findAgentEventAppliedAction,createAgentEventActionRecorder,isHITLEnabled,@@ -61,6 +67,12 @@ const {releaseAgentEventActorAction,hasAgentEventActorActionAdmission,getAgentEventActorReceipt,+ getAgentEventActorDetachedAction,+ reserveAgentEventActorDetachedAction,+ markAgentEventActorDetachedActionRunning,+ settleAgentEventActorDetachedAction,+ claimAgentEventActorSuspension,+ settleAgentEventActorSuspension,isAgentTriggerPrincipalActive,isSubagentOwnerAdmissible,} = require('~/models');@@ -1310,12 +1322,21 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit* idempotency key that owns generation admission. Ignore mismatched or* direct-chat metadata rather than letting callers relabel another run. */const rawAgentEventDelivery = req.body?.agentEventDelivery;+ const internalDetachedCompletion = parseAgentEventActorDetachedCompletion(+ rawAgentEventDelivery?.internalCompletion,+ );+ const isInternalDetachedCompletion =+ internalDetachedCompletion != null &&+ rawAgentEventDelivery?.deliveryKey === clientRequestId &&+ rawAgentEventDelivery?.event?.type === EVENT_ACTOR_DETACHED_COMPLETION_TYPE &&+ rawAgentEventDelivery?.event?.source?.type === 'internal' &&+ rawAgentEventDelivery?.event?.source?.id === EVENT_ACTOR_DETACHED_COMPLETION_SOURCE;const agentEventDelivery =isTriggerContinuation &&isBoundEventContinuation &&rawAgentEventDelivery != null &&typeof rawAgentEventDelivery === 'object' &&- rawAgentEventDelivery.deliveryKey === clientRequestId+ (rawAgentEventDelivery.deliveryKey === clientRequestId || isInternalDetachedCompletion)? rawAgentEventDelivery: undefined;req._agentEventTriggerProjection = getAgentEventTriggerProjection(agentEventDelivery);@@ -1367,9 +1388,20 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitisTemporary: req._agentEventBindingRetention?.isTemporary ?? req.body?.isTemporary,...(agentEventDelivery != null && {agentEventDeliveryKey: agentEventDelivery.deliveryKey,+ ...(internalDetachedCompletion == null+ ? {}+ : {+ agentEventInvocationKey: internalDetachedCompletion.invocationId,+ agentEventInvocationGenerationCreatedAt:+ internalDetachedCompletion.generationCreatedAt,+ }),agentEventBindingId: boundEventBindingId,...(agentEventDelivery.expectedAction != null && {agentEventExpectedAction: agentEventDelivery.expectedAction,+ ...(GenerationJobManager.isRedis &&+ isAgentEventActorDetachedActionProducerEnabled() && {+ agentEventDetachedActionProducerRequired: true,+ }),}),}),...(isRegenerate && { isRegenerate: true }),@@ -1691,15 +1723,6 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitconst eventActorMayPause =isHITLEnabled(agentsConfig?.toolApproval) ||eventActorAgents.some(agentRequestsAskUserQuestion);- const expectedActionToolName = agentEventDelivery?.expectedAction?.toolName;- const eventActorActionMayDetach =- typeof expectedActionToolName === 'string' &&- eventActorAgents.some((agent) =>- (agent.backgroundToolNames ?? []).some(- (name) =>- name === expectedActionToolName || name.startsWith(`${expectedActionToolName}_mcp_`),- ),- );const turnExecutionPlan = resolveAgentTurnExecutionPlan({conversationId,parentMessageId,@@ -1731,7 +1754,6 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit* `/resume` replica from consuming a signed suspension it cannot claim. */durableEventActorSuspensions: generationProtocolVersion >= GENERATION_PROTOCOL_V2,checkpointerType: agentsConfig?.checkpointer?.type,- expectedActionMayDetach: eventActorActionMayDetach,});// Resolve title timing from the public agents endpoint first, then fall@@ -2061,67 +2083,213 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitconst eventActorActionRecorder = usesCheckpointStrategy? createAgentEventActionRecorder(turnExecutionPlan.expectedAction): undefined;+ const actorInvocationId = internalDetachedCompletion?.invocationId ?? eventTaskId;+ const eventActorDetachedAction =+ usesCheckpointStrategy &&+ turnExecutionPlan.expectedAction != null &&+ turnExecutionPlan.binding != null+ ? createAgentEventActorDetachedActionLifecycle(+ {+ user: userId,+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),+ bindingId: turnExecutionPlan.binding.bindingId,+ conversationId,+ generationCreatedAt:+ internalDetachedCompletion?.generationCreatedAt ?? jobCreatedAt,+ turnCreatedAt: jobCreatedAt,+ invocationId: actorInvocationId,+ expectedAction: turnExecutionPlan.expectedAction,+ },+ {+ reserveAgentEventActorDetachedAction,+ markAgentEventActorDetachedActionRunning,+ settleAgentEventActorDetachedAction,+ producerEnabled: () =>+ GenerationJobManager.isRedis &&+ isAgentEventActorDetachedActionProducerEnabled(),+ persistTerminalEvidence: async (evidence) => {+ const persisted =+ await GenerationJobManager.persistAgentEventDetachedTerminalEvidence(+ streamId,+ jobCreatedAt,+ evidence,+ );+ if (!persisted) {+ throw new Error(+ 'Detached Event Actor terminal retry evidence could not be staged',+ );+ }+ },+ /** Retry immediately when the generation already reached its+ * terminal host-action fence. The same durable marker is+ * recovered across replicas and restarts by the existing+ * GenerationJobManager sweep. */+ onTerminal: async () => {+ await GenerationJobManager.retryTerminalHostAction(streamId, jobCreatedAt);+ },+ },+ )+ : undefined;+ req._agentEventDetachedActionLifecycle = eventActorDetachedAction;if (eventActorActionRecorder != null) {req._agentEventActionObserver = eventActorActionRecorder.observeToolEnd;}+ let internalDetachedAction;+ let internalDetachedSuspension;+ if (usesCheckpointStrategy && isInternalDetachedCompletion) {+ const snapshot = await getAgentEventActorSnapshot({+ user: userId,+ conversationId,+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),+ });+ internalDetachedSuspension = snapshot?.suspension;+ internalDetachedAction = await getAgentEventActorDetachedAction({+ deliveryKey: internalDetachedCompletion.invocationId,+ user: userId,+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),+ bindingId: turnExecutionPlan.binding.bindingId,+ conversationId,+ generationCreatedAt: internalDetachedCompletion.generationCreatedAt,+ });+ if (+ internalDetachedSuspension?.kind !== 'internal_completion' ||+ internalDetachedSuspension.status !== 'pending' ||+ internalDetachedSuspension.actionId !== internalDetachedCompletion.taskId ||+ internalDetachedSuspension.suspension.invocation.invocationId !==+ internalDetachedCompletion.invocationId ||+ internalDetachedAction == null ||+ internalDetachedAction.taskId !== internalDetachedCompletion.taskId ||+ internalDetachedAction.idempotencyKey !== internalDetachedCompletion.idempotencyKey ||+ !['succeeded', 'failed', 'cancelled'].includes(internalDetachedAction.status)+ ) {+ throw Object.assign(+ new Error('The detached Event Actor completion is no longer current'),+ { code: 'EVENT_ACTOR_NOT_READY', status: 409 },+ );+ }+ }+ const readAppliedEventAction = () =>+ eventActorActionRecorder.read() ??+ (internalDetachedAction?.status === 'succeeded'+ ? {+ toolName: internalDetachedAction.toolName,+ toolCallId: internalDetachedAction.toolCallId,+ }+ : undefined) ??+ findAgentEventAppliedAction(+ turnExecutionPlan.expectedAction,+ client?.run?.getRunSteps?.() ?? [],+ client?.contentParts ?? [],+ );+ const actorDependencies = {+ getSnapshot: getAgentEventActorSnapshot,+ commitState: commitAgentEventActorState,+ storeSuspension: storeAgentEventActorSuspension,+ claimSuspension: claimAgentEventActorSuspension,+ settleSuspension: settleAgentEventActorSuspension,+ recordReconciliation: recordAgentEventActorReconciliation,+ resolveReconciliation: resolveAgentEventActorReconciliation,+ admitAction: admitAgentEventActorAction,+ releaseAction: releaseAgentEventActorAction,+ hasActionAdmission: hasAgentEventActorActionAdmission,+ getReceipt: getAgentEventActorReceipt,+ clearReconciliation: clearAgentEventActorReconciliation,+ };const sendPromise = usesCheckpointStrategy- ? executeAgentEventActor(- {- user: userId,- ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),- conversationId,- bindingId: turnExecutionPlan.binding.bindingId,- invocationId: eventTaskId,- event: agentEventDelivery.event,- expectedAction: turnExecutionPlan.expectedAction,- signal: job.abortController.signal,- checkpointer: req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,- resolveContext: (state) => client.prepareEventActorContext(state),- readResultContext: () => client.getEventActorContext(),- invoke: async (actorContext) => {- client.checkpointNamespace = actorContext.checkpointNamespace;- client.eventActorCheckpointId = actorContext.checkpointId;- client.eventActorInvocationId = actorContext.invocationId;- client.eventActorContinuation = actorContext.continuation;- return client.sendMessage(text, messageOptions);- },- readAppliedAction: () =>- eventActorActionRecorder.read() ??- findAgentEventAppliedAction(- turnExecutionPlan.expectedAction,- client?.run?.getRunSteps?.() ?? [],- client?.contentParts ?? [],- ),- readSuspension: () => client.readEventActorSuspension(),- },- {- getSnapshot: getAgentEventActorSnapshot,- commitState: commitAgentEventActorState,- storeSuspension: storeAgentEventActorSuspension,- recordReconciliation: recordAgentEventActorReconciliation,- resolveReconciliation: resolveAgentEventActorReconciliation,- admitAction: admitAgentEventActorAction,- releaseAction: releaseAgentEventActorAction,- hasActionAdmission: hasAgentEventActorActionAdmission,- getReceipt: getAgentEventActorReceipt,- clearReconciliation: clearAgentEventActorReconciliation,- },+ ? (isInternalDetachedCompletion+ ? resumeAgentEventActor(+ {+ user: userId,+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),+ conversationId,+ bindingId: turnExecutionPlan.binding.bindingId,+ suspension: internalDetachedSuspension.suspension,+ resumeAttemptId: clientRequestId,+ resumeValue: {+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ taskId: internalDetachedAction.taskId,+ status: internalDetachedAction.status,+ ...(internalDetachedAction.result == null+ ? {}+ : { result: internalDetachedAction.result }),+ ...(internalDetachedAction.error == null+ ? {}+ : { error: internalDetachedAction.error }),+ },+ signal: job.abortController.signal,+ checkpointer: req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,+ expectedAction: turnExecutionPlan.expectedAction,+ resume: async (actorContext) => {+ client.checkpointNamespace = actorContext.checkpointNamespace;+ client.eventActorCheckpointId = actorContext.checkpointId;+ client.eventActorInvocationId = actorContext.invocationId;+ client.eventActorContinuation = actorContext.continuation;+ return client.sendMessage(text, messageOptions);+ },+ readAppliedAction: readAppliedEventAction,+ readSuspension: () =>+ eventActorDetachedAction?.readSuspension() ??+ client.readEventActorSuspension(),+ readResultContext: () => client.getEventActorContext(),+ },+ actorDependencies,+ )+ : executeAgentEventActor(+ {+ user: userId,+ ...(eventActorTenantId == null ? {} : { tenantId: eventActorTenantId }),+ conversationId,+ bindingId: turnExecutionPlan.binding.bindingId,+ invocationId: actorInvocationId,+ event: agentEventDelivery.event,+ expectedAction: turnExecutionPlan.expectedAction,+ signal: job.abortController.signal,+ checkpointer: req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,+ resolveContext: (state) => client.prepareEventActorContext(state),+ readResultContext: () => client.getEventActorContext(),+ invoke: async (actorContext) => {+ client.checkpointNamespace = actorContext.checkpointNamespace;+ client.eventActorCheckpointId = actorContext.checkpointId;+ client.eventActorInvocationId = actorContext.invocationId;+ client.eventActorContinuation = actorContext.continuation;+ return client.sendMessage(text, messageOptions);+ },+ readAppliedAction: readAppliedEventAction,+ readSuspension: () =>+ eventActorDetachedAction?.readSuspension() ??+ client.readEventActorSuspension(),+ },+ actorDependencies,+ )).then(async ({ value, execution }) => {if (execution.status === 'applied') {appliedEventActor = {- invocationId: eventTaskId,+ invocationId: actorInvocationId,actionAdmitted: typeof admitAgentEventActorAction === 'function',checkpoint: execution.head.checkpoint,action: execution.result.action,};} else if (execution.status === 'suspended') {- if (!(await client.publishStagedApproval(execution.suspension))) {+ const suspensionKind = eventActorDetachedAction?.readSuspension()?.kind;+ if (suspensionKind === 'internal_completion') {+ await GenerationJobManager.updateMetadata(+ streamId,+ {+ agentEventSuspension: {+ version: execution.suspension.version,+ suspensionId: execution.suspension.suspensionId,+ attempt: execution.suspension.attempt,+ },+ },+ jobCreatedAt,+ );+ } else if (!(await client.publishStagedApproval(execution.suspension))) {throw new Error('Event actor suspension could not be projected to its job');}}logger.info('[event-actor] Bound child event completed', {conversationId,- invocationId: eventTaskId,+ invocationId: actorInvocationId,status: execution.status,continuation: execution.continuation,});diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.jsindex f6644de811..c7fe2d697e 100644--- a/api/server/controllers/agents/resume.js+++ b/api/server/controllers/agents/resume.js@@ -42,6 +42,8 @@ const {isAgentEventRetentionActive,resumeAgentEventActor,createAgentEventActionRecorder,+ createAgentEventActorDetachedActionLifecycle,+ isAgentEventActorDetachedActionProducerEnabled,findAgentEventAppliedAction,} = require('@librechat/api');const { disposeClient } = require('~/server/cleanup');@@ -60,6 +62,7 @@ const {getActions,getUserMemories,getRoleByName,+ getAgentTriggerDelivery,isSubagentOwnerAdmissible,getAgentEventActorSnapshot,commitAgentEventActorState,@@ -68,6 +71,9 @@ const {settleAgentEventActorSuspension,recordAgentEventActorReconciliation,completeAgentEventActorLegacyTurn,+ reserveAgentEventActorDetachedAction,+ markAgentEventActorDetachedActionRunning,+ settleAgentEventActorDetachedAction,} = require('~/models');const {acquireEventChildGenerationLease,@@ -1196,6 +1202,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)let releaseEventChildLease;let eventLeaseTransferredToRun = false;let durableEventActorSuspension;+ let durableEventActorHandlingGenerationCreatedAt;+ let durableEventActorRequiresDetachedProducer = false;let eventActorResumePromise;let eventActorStartGate;let eventActorContinuationStarted = false;@@ -1330,6 +1338,14 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)suspensionRecord.suspension.attempt === suspensionProjection.attempt) {durableEventActorSuspension = suspensionRecord.suspension;+ durableEventActorHandlingGenerationCreatedAt =+ suspensionRecord.handlingGenerationCreatedAt;+ durableEventActorRequiresDetachedProducer =+ job.metadata.agentEventDetachedActionProducerRequired === true ||+ (suspensionRecord.handlingGenerationCreatedAt != null &&+ job.metadata.agentEventExpectedAction != null) ||+ job.metadata.agentEventInvocationKey != null ||+ suspensionRecord.kind === 'internal_completion';const signedExpectedAction = getSuspendedEventActorExpectedAction(durableEventActorSuspension,);@@ -1367,6 +1383,26 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle));}}+ if (+ durableEventActorSuspension != null &&+ durableEventActorRequiresDetachedProducer &&+ (!GenerationJobManager.isRedis || !isAgentEventActorDetachedActionProducerEnabled())+ ) {+ const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);+ await rollbackUnconsumedScheduleClaim(currentJob);+ await releaseScheduleFence();+ await decrementPendingRequest(userId);+ res.set('Retry-After', '1');+ return sendGenerationJson(+ res,+ 503,+ {+ code: 'EVENT_ACTOR_RESUME_CAPABILITY_UNAVAILABLE',+ error: 'A durable Event Actor resume worker is temporarily unavailable',+ },+ generationProtocolVersion,+ );+ }}// Atomically claim the resume. The single winner drives the run; a racing second@@ -1401,6 +1437,65 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)const claimGate = deferred();eventActorStartGate = deferred();const expectedAction = getSuspendedEventActorExpectedAction(durableEventActorSuspension);+ const actorInvocationId =+ job.metadata.agentEventInvocationKey ?? job.metadata.agentEventDeliveryKey;+ let actorInvocationGenerationCreatedAt =+ job.metadata.agentEventInvocationGenerationCreatedAt ??+ durableEventActorHandlingGenerationCreatedAt ??+ (job.metadata.agentEventInvocationKey == null ? job.createdAt : undefined);+ if (+ actorInvocationGenerationCreatedAt == null &&+ job.metadata.agentEventInvocationKey != null+ ) {+ const originalDelivery = await getAgentTriggerDelivery(+ job.metadata.agentEventInvocationKey,+ );+ actorInvocationGenerationCreatedAt = originalDelivery?.handling?.generationCreatedAt;+ }+ if (+ durableEventActorRequiresDetachedProducer &&+ actorInvocationId != null &&+ Number.isSafeInteger(actorInvocationGenerationCreatedAt) &&+ req._agentEventBindingId != null+ ) {+ req._agentEventDetachedActionLifecycle = createAgentEventActorDetachedActionLifecycle(+ {+ user: userId,+ ...(req._agentEventBindingTenantId == null+ ? {}+ : { tenantId: req._agentEventBindingTenantId }),+ bindingId: req._agentEventBindingId,+ conversationId,+ generationCreatedAt: actorInvocationGenerationCreatedAt,+ turnCreatedAt: job.createdAt,+ invocationId: actorInvocationId,+ expectedAction,+ },+ {+ reserveAgentEventActorDetachedAction,+ markAgentEventActorDetachedActionRunning,+ settleAgentEventActorDetachedAction,+ producerEnabled: () =>+ GenerationJobManager.isRedis && isAgentEventActorDetachedActionProducerEnabled(),+ persistTerminalEvidence: async (evidence) => {+ const persisted =+ await GenerationJobManager.persistAgentEventDetachedTerminalEvidence(+ streamId,+ job.createdAt,+ evidence,+ );+ if (!persisted) {+ throw new Error(+ 'Detached Event Actor terminal retry evidence could not be staged',+ );+ }+ },+ onTerminal: async () => {+ await GenerationJobManager.retryTerminalHostAction(streamId, job.createdAt);+ },+ },+ );+ }eventActorActionRecorder = createAgentEventActionRecorder(expectedAction);req._agentEventActionObserver = eventActorActionRecorder.observeToolEnd;eventActorResumePromise = resumeAgentEventActor(@@ -1454,7 +1549,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)client?.contentParts ?? [],{ userSubmittedMessageFieldPaths },),- readSuspension: () => client?.readEventActorSuspension(),+ readSuspension: () =>+ req._agentEventDetachedActionLifecycle?.readSuspension() ??+ client?.readEventActorSuspension(),readResultContext: () => client?.getEventActorContext(),},{@@ -1749,7 +1846,20 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)});const actorResult = await eventActorResumePromise;if (actorResult.execution.status === 'suspended') {- if (!(await client.publishStagedApproval(actorResult.execution.suspension))) {+ const suspensionKind = req._agentEventDetachedActionLifecycle?.readSuspension()?.kind;+ if (suspensionKind === 'internal_completion') {+ await GenerationJobManager.updateMetadata(+ streamId,+ {+ agentEventSuspension: {+ version: actorResult.execution.suspension.version,+ suspensionId: actorResult.execution.suspension.suspensionId,+ attempt: actorResult.execution.suspension.attempt,+ },+ },+ job.createdAt,+ );+ } else if (!(await client.publishStagedApproval(actorResult.execution.suspension))) {throw new Error('Re-paused event actor suspension could not be projected to its job');}} else if (actorResult.execution.status === 'applied') {diff --git a/api/server/experimental.js b/api/server/experimental.jsindex edd3d84e02..b4083feeda 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -46,6 +46,7 @@ const createValidateImageRequest = require('./middleware/validateImageRequest');const { startExpiredFileSweep } = require('./services/Files/process');const { initializeGitHubSkillSync } = require('./services/Skills/sync');const { initializeAgentTriggerService } = require('./services/Agents/triggers');+const { resumeAgentEventDetachedAction } = require('./services/Agents/detachedActionResume');const {recordExpiredScheduleApproval,initializeScheduleErasureSweep,@@ -295,7 +296,9 @@ if (cluster.isMaster) {// its durable run when the generic approval runtime expires it.GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);GenerationJobManager.setTerminalHostActionHandler(- createAgentEventTerminalHandler(agentEventMethods),+ createAgentEventTerminalHandler(agentEventMethods, {+ resumeDetachedAction: resumeAgentEventDetachedAction,+ }),);GenerationJobManager.initialize();/**diff --git a/api/server/index.js b/api/server/index.jsindex 8796f3dbca..85ff068b5e 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -67,6 +67,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilitiesconst createValidateImageRequest = require('./middleware/validateImageRequest');const { initializeGitHubSkillSync } = require('./services/Skills/sync');const { initializeAgentTriggerService } = require('./services/Agents/triggers');+const { resumeAgentEventDetachedAction } = require('./services/Agents/detachedActionResume');const { initializeScheduleEngine, recordExpiredScheduleApproval } = require('./services/Schedules');const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');const { startExpiredFileSweep } = require('./services/Files/process');@@ -128,7 +129,9 @@ const configureGenerationStreams = () => {});GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);GenerationJobManager.setTerminalHostActionHandler(- createAgentEventTerminalHandler(agentEventMethods),+ createAgentEventTerminalHandler(agentEventMethods, {+ resumeDetachedAction: resumeAgentEventDetachedAction,+ }),);GenerationJobManager.initialize();// Stop active generations and close their SSE streams while the HTTP server drains.diff --git a/api/server/services/Agents/detachedActionResume.js b/api/server/services/Agents/detachedActionResume.jsnew file mode 100644index 0000000000..444584b9f4--- /dev/null+++ b/api/server/services/Agents/detachedActionResume.js@@ -0,0 +1,12 @@+const { createAgentEventDetachedResumeHandler } = require('@librechat/api');+const methods = require('~/models');++const resumeAgentEventDetachedAction = createAgentEventDetachedResumeHandler({+ getAgentTriggerDelivery: methods.getAgentTriggerDelivery,+ /** Keep server module construction lazy while the typed package owns the+ * continuation protocol and owner validation. */+ enqueueAgentTrigger: (envelope, options) =>+ require('./triggers').enqueueAgentTrigger(envelope, options),+});++module.exports = { resumeAgentEventDetachedAction };diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.jsindex 02e9411929..f6269e2cba 100644--- a/api/server/services/Agents/triggers.js+++ b/api/server/services/Agents/triggers.js@@ -3,6 +3,7 @@ const {createAgentEventContinueResolver,createSubagentCompletionWakeupResolver,GenerationJobManager,+ isAgentEventActorDetachedActionProducerEnabled,} = require('@librechat/api');const methods = require('~/models');@@ -14,6 +15,8 @@ const completionResolver = createSubagentCompletionWakeupResolver({const service = createAgentTriggerService({methods,isPrincipalActive: methods.isAgentTriggerPrincipalActive,+ supportsDetachedActionCompletion: () =>+ GenerationJobManager.isRedis && isAgentEventActorDetachedActionProducerEnabled(),prepareContinue: createAgentEventContinueResolver({methods,getGenerationJob: (conversationId) => GenerationJobManager.getJob(conversationId),diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.jsnew file mode 100644index 0000000000..d824e68182--- /dev/null+++ b/api/server/services/Agents/triggers.spec.js@@ -0,0 +1,54 @@+const mockCreateAgentTriggerService = jest.fn();+const mockIsProducerEnabled = jest.fn();+const mockGenerationJobManager = {+ isRedis: true,+ getJob: jest.fn(),+};++jest.mock('@librechat/api', () => ({+ createAgentTriggerService: (...args) => mockCreateAgentTriggerService(...args),+ createAgentEventContinueResolver: jest.fn(() => jest.fn()),+ createSubagentCompletionWakeupResolver: jest.fn(() => jest.fn()),+ GenerationJobManager: mockGenerationJobManager,+ isAgentEventActorDetachedActionProducerEnabled: () => mockIsProducerEnabled(),+}));++jest.mock('~/models', () => ({+ isAgentTriggerPrincipalActive: jest.fn(),+}));++describe('agent trigger service composition', () => {+ beforeEach(() => {+ jest.resetModules();+ jest.clearAllMocks();+ mockGenerationJobManager.isRedis = true;+ mockIsProducerEnabled.mockReturnValue(true);+ mockCreateAgentTriggerService.mockReturnValue({+ initialize: jest.fn(),+ stop: jest.fn(),+ dispatch: jest.fn(),+ enqueue: jest.fn(),+ getDelivery: jest.fn(),+ getDeliveryStatus: jest.fn(),+ getDeadLetters: jest.fn(),+ requeue: jest.fn(),+ drainUser: jest.fn(),+ prepareUserPurge: jest.fn(),+ cancelUserPurge: jest.fn(),+ purgeUser: jest.fn(),+ });+ });++ it('advertises detached completion capability only after Redis and producer activation', () => {+ require('./triggers');+ const supportsDetachedActionCompletion =+ mockCreateAgentTriggerService.mock.calls[0][0].supportsDetachedActionCompletion;++ expect(supportsDetachedActionCompletion()).toBe(true);+ mockIsProducerEnabled.mockReturnValue(false);+ expect(supportsDetachedActionCompletion()).toBe(false);+ mockIsProducerEnabled.mockReturnValue(true);+ mockGenerationJobManager.isRedis = false;+ expect(supportsDetachedActionCompletion()).toBe(false);+ });+});diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.jsindex 41f18cabc3..a60a17dc7a 100644--- a/api/server/services/Endpoints/agents/initialize.js+++ b/api/server/services/Endpoints/agents/initialize.js@@ -388,6 +388,18 @@ const initializeClient = async ({});},toolEndCallback,+ /** Bound later by request.js once the authenticated Event Actor owner and+ * generation fence are known. Ordinary background calls remain unchanged. */+ eventActorDetachedAction: {+ reserve: (input) =>+ req._agentEventDetachedActionLifecycle?.reserve(input) ??+ Promise.resolve({ status: 'ignored' }),+ markRunning: (input) =>+ req._agentEventDetachedActionLifecycle?.markRunning(input) ?? Promise.resolve(false),+ settle: (input) =>+ req._agentEventDetachedActionLifecycle?.settle(input) ?? Promise.resolve(false),+ wake: (input) => req._agentEventDetachedActionLifecycle?.wake(input) ?? Promise.resolve(),+ },persistBackgroundCodeResult: createBackgroundCodeResultHandler({req,updateToolCallResult: db.updateToolCallResult,diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.tsindex 2a32182ee0..22cb2c5519 100644--- a/packages/api/src/agents/background.spec.ts+++ b/packages/api/src/agents/background.spec.ts@@ -1100,6 +1100,80 @@ describe('BackgroundTaskRegistryClass', () => {expect(atCapacity).toBe(true);});+ it('holds capacity permits before task registration and releases them explicitly', () => {+ const registry = new BackgroundTaskRegistryClass();+ const permits = Array.from({ length: 10 }, (_, index) =>+ registry.reserveCapacity({+ userId: 'u1',+ conversationId: 'c-permits',+ toolCallId: `call_${index}`,+ runId: 'run-1',+ }),+ );+ expect(permits.every((result) => 'permit' in result)).toBe(true);+ expect(+ registry.reserveCapacity({+ userId: 'u1',+ conversationId: 'c-permits',+ toolCallId: 'call_rejected',+ runId: 'run-1',+ }),+ ).toEqual({ atCapacity: true });+ const first = permits[0];+ if (!('permit' in first)) {+ throw new Error('expected capacity permit');+ }+ registry.releaseCapacity(first.permit);+ const replacement = registry.reserveCapacity({+ userId: 'u1',+ conversationId: 'c-permits',+ toolCallId: 'call_replacement',+ runId: 'run-1',+ });+ if (!('permit' in replacement)) {+ throw new Error('expected replacement permit');+ }+ const created = registry.create({+ userId: 'u1',+ conversationId: 'c-permits',+ toolCallId: 'call_replacement',+ toolName: 'search_mcp_docs',+ runId: 'run-1',+ capacityPermit: replacement.permit,+ });+ expect('atCapacity' in created).toBe(false);+ });++ it('retains an in-flight capacity permit until durable reservation completes', () => {+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_787_000_000_000);+ const registry = new BackgroundTaskRegistryClass();+ const admission = registry.reserveCapacity({+ userId: 'u1',+ conversationId: 'c-slow-reservation',+ toolCallId: 'call_slow',+ runId: 'run-1',+ });+ if (!('permit' in admission)) {+ throw new Error('expected capacity permit');+ }++ /** Cross both the former one-minute permit timeout and idle-bucket TTL.+ * A slow durable reservation still owns this slot until its caller+ * consumes or releases it. */+ now.mockReturnValue(1_787_000_000_000 + 7 * 60 * 60 * 1000);+ const created = registry.create({+ userId: 'u1',+ conversationId: 'c-slow-reservation',+ toolCallId: 'call_slow',+ toolName: 'search_mcp_docs',+ runId: 'run-1',+ capacityPermit: admission.permit,+ });++ expect('atCapacity' in created).toBe(false);+ now.mockRestore();+ });+it('evicts oldest settled tasks instead of blocking when the total cap is full', () => {const registry = new BackgroundTaskRegistryClass();for (let i = 0; i < 200; i++) {diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.tsindex 4b1daa511f..13c15dc687 100644--- a/packages/api/src/agents/background.ts+++ b/packages/api/src/agents/background.ts@@ -634,9 +634,19 @@ interface TaskBucket {tasks: Map<string, BackgroundTask>;/** toolCallId -> taskId, for dispatch idempotency across graph re-execution. */byToolCall: Map<string, string>;+ /** Caller-owned local permits acquired before a caller persists external+ * launch authority. They prevent capacity rejection from creating a durable+ * action that was definitely never launched. */+ capacityPermits: Map<string, { dedupeKey: string }>;lastAccess: number;}+export interface BackgroundTaskCapacityPermit {+ id: string;+ userId: string;+ conversationId: string;+}+const COMPLETED_TASK_TTL_MS = 60 * 60 * 1000;const IDLE_BUCKET_TTL_MS = 6 * 60 * 60 * 1000;/** Max wall-clock a task may stay `running` before being reaped as timed-out,@@ -742,7 +752,7 @@ export class BackgroundTaskRegistryClass {}this.lastGlobalSweepAt = now;for (const [bucketKey, bucket] of this.buckets) {- if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS) {+ if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) {this.buckets.delete(bucketKey);continue;}@@ -754,13 +764,77 @@ export class BackgroundTaskRegistryClass {const bucketKey = this.key(userId, conversationId);let bucket = this.buckets.get(bucketKey);if (!bucket) {- bucket = { tasks: new Map(), byToolCall: new Map(), lastAccess: now };+ bucket = {+ tasks: new Map(),+ byToolCall: new Map(),+ capacityPermits: new Map(),+ lastAccess: now,+ };this.buckets.set(bucketKey, bucket);}bucket.lastAccess = now;return bucket;}+ private dedupeKey(params: { toolCallId: string; runId?: string; agentId?: string }): string {+ return `${params.agentId ?? ''}::${params.runId ?? ''}::${params.toolCallId}`;+ }++ private runningCount(bucket: TaskBucket): number {+ let running = 0;+ for (const task of bucket.tasks.values()) {+ if (task.status === 'running') {+ running++;+ }+ }+ return running;+ }++ /** Acquires process-local capacity before a caller persists launch authority.+ * The synchronous permit closes the capacity-rejection crash window without+ * making ordinary background tasks durable. */+ reserveCapacity(params: {+ userId: string;+ conversationId: string;+ toolCallId: string;+ runId?: string;+ agentId?: string;+ }):+ | { permit: BackgroundTaskCapacityPermit }+ | { task: BackgroundTask; isNew: false }+ | { atCapacity: true } {+ const now = Date.now();+ this.sweep(now);+ const bucket = this.getBucket(params.userId, params.conversationId, now);+ this.sweepBucketTasks(bucket, now);+ const dedupeKey = this.dedupeKey(params);+ const existingId = bucket.byToolCall.get(dedupeKey);+ const existing = existingId == null ? undefined : bucket.tasks.get(existingId);+ if (existing != null) {+ return { task: existing, isNew: false };+ }+ if (this.runningCount(bucket) + bucket.capacityPermits.size >= MAX_RUNNING_PER_BUCKET) {+ return { atCapacity: true };+ }+ const permit: BackgroundTaskCapacityPermit = {+ id: randomUUID(),+ userId: params.userId,+ conversationId: params.conversationId,+ };+ /** The permit is owned by the in-flight caller until it is consumed or+ * explicitly released. Expiring it by wall clock could strand a durable+ * reservation when MongoDB is slow; process death already clears local+ * permits without pretending the external launch happened. */+ bucket.capacityPermits.set(permit.id, { dedupeKey });+ return { permit };+ }++ releaseCapacity(permit: BackgroundTaskCapacityPermit): void {+ this.buckets+ .get(this.key(permit.userId, permit.conversationId))+ ?.capacityPermits.delete(permit.id);+ }+/*** Registers a task for a tool call. Returns the existing task (and* `isNew: false`) only when the SAME run re-dispatched the same `toolCallId`@@ -774,6 +848,7 @@ export class BackgroundTaskRegistryClass {* task and hand back a stale/foreign result instead of executing.*/create(params: {+ taskId?: string;userId: string;conversationId: string;toolCallId: string;@@ -785,29 +860,41 @@ export class BackgroundTaskRegistryClass {* never settle (reaped as timed out) still take the marker/heal path* instead of leaving the original card on "running" forever. */harvestStarted?: boolean;+ capacityPermit?: BackgroundTaskCapacityPermit;}): { task: BackgroundTask; isNew: boolean } | { atCapacity: true } {const now = Date.now();this.sweep(now);const bucket = this.getBucket(params.userId, params.conversationId, now);this.sweepBucketTasks(bucket, now);- const dedupeKey = `${params.agentId ?? ''}::${params.runId ?? ''}::${params.toolCallId}`;+ const dedupeKey = this.dedupeKey(params);const existingId = bucket.byToolCall.get(dedupeKey);if (existingId) {const existing = bucket.tasks.get(existingId);if (existing) {+ if (params.capacityPermit != null) {+ this.releaseCapacity(params.capacityPermit);+ }return { task: existing, isNew: false };}}- let running = 0;- for (const task of bucket.tasks.values()) {- if (task.status === 'running') {- running++;+ if (params.capacityPermit != null) {+ const permit = bucket.capacityPermits.get(params.capacityPermit.id);+ if (+ params.capacityPermit.userId !== params.userId ||+ params.capacityPermit.conversationId !== params.conversationId ||+ permit?.dedupeKey !== dedupeKey+ ) {+ throw new Error('Background task capacity permit is stale');}+ bucket.capacityPermits.delete(params.capacityPermit.id);}/** Only *running* tasks gate dispatch. */- if (running >= MAX_RUNNING_PER_BUCKET) {+ if (+ params.capacityPermit == null &&+ this.runningCount(bucket) + bucket.capacityPermits.size >= MAX_RUNNING_PER_BUCKET+ ) {return { atCapacity: true };}/** The total-tasks cap bounds memory but must NOT block new work: evict the@@ -829,7 +916,7 @@ export class BackgroundTaskRegistryClass {}const task: BackgroundTask = {- id: randomUUID(),+ id: params.taskId ?? randomUUID(),toolName: params.toolName,toolCallId: params.toolCallId,messageId: params.messageId,@@ -1050,7 +1137,9 @@ export class BackgroundTaskRegistryClass {export const backgroundTaskRegistry = new BackgroundTaskRegistryClass();/** Content for the synthetic ToolMessage returned when a call is backgrounded. */-export function buildBackgroundHandleContent(task: BackgroundTask): string {+export function buildBackgroundHandleContent(+ task: Pick<BackgroundTask, 'id' | 'toolName' | 'status'>,+): string {return JSON.stringify({background_task_id: task.id,tool: task.toolName,diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.tsindex 7ac1f089d7..5de6e55ebd 100644--- a/packages/api/src/agents/handlers.background.spec.ts+++ b/packages/api/src/agents/handlers.background.spec.ts@@ -84,6 +84,350 @@ const runBatch = async (};describe('createToolExecuteHandler — background tool calls', () => {+ it('persists an Event Actor launch before invoking and terminal evidence before wakeup', async () => {+ const events: string[] = [];+ let finishTool: ((value: { content: string }) => void) | undefined;+ const tool = {+ name: 'submit_move_mcp_chess',+ description: 'submits a move',+ schema: z.object({ move: z.string() }),+ invoke: () => {+ events.push('invoke');+ return new Promise<{ content: string }>((resolve) => {+ finishTool = resolve;+ });+ },+ } as unknown as StructuredToolInterface;+ let reservations = 0;+ const eventActorDetachedAction = {+ reserve: jest.fn(async () => {+ events.push('reserve');+ reservations += 1;+ return {+ status: reservations === 1 ? ('reserved' as const) : ('replay' as const),+ taskId: 'event-task-stable-1',+ idempotencyKey: 'a'.repeat(64),+ };+ }),+ markRunning: jest.fn(async () => {+ events.push('running');+ return true;+ }),+ settle: jest.fn(async () => {+ events.push('terminal');+ return true;+ }),+ wake: jest.fn(async () => {+ events.push('wake');+ }),+ };+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ eventActorDetachedAction,+ });+ const request = {+ toolCalls: [+ {+ id: 'call-event-detached',+ name: tool.name,+ args: { move: 'e4', run_in_background: true },+ },+ ],+ agentId: 'agent-player',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },+ };++ const first = await runBatch(handler, request);++ expect(JSON.parse(first[0].content)).toMatchObject({+ status: 'running',+ background_task_id: 'event-task-stable-1',+ });+ expect(events).toEqual(['reserve', 'invoke', 'running']);+ finishTool?.({ content: 'move accepted' });+ await flushMicrotasks();+ await flushMicrotasks();+ expect(events).toEqual(['reserve', 'invoke', 'running', 'terminal', 'wake']);+ expect(eventActorDetachedAction.reserve).toHaveBeenCalledWith(+ expect.objectContaining({ turnId: 'event-response:' }),+ );++ const replay = await runBatch(handler, request);++ expect(JSON.parse(replay[0].content).background_task_id).toBe('event-task-stable-1');+ expect(events).toEqual(['reserve', 'invoke', 'running', 'terminal', 'wake', 'reserve']);+ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(+ expect.objectContaining({+ taskId: 'event-task-stable-1',+ status: 'succeeded',+ result: 'move accepted',+ }),+ );+ });++ it('records filtered detached output as successful side-effect evidence', async () => {+ const protectedValue = 'Authorization: Bearer detached-secret';+ const tool = {+ name: 'submit_move_mcp_chess',+ description: 'submits a move',+ schema: z.object({ move: z.string() }),+ invoke: jest.fn(async () => ({ content: protectedValue })),+ } as unknown as StructuredToolInterface;+ const eventActorDetachedAction = {+ reserve: jest.fn(async () => ({+ status: 'reserved' as const,+ taskId: 'event-task-filtered',+ idempotencyKey: 'e'.repeat(64),+ })),+ markRunning: jest.fn(async () => true),+ settle: jest.fn(async () => true),+ wake: jest.fn(async () => undefined),+ };+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ eventActorDetachedAction,+ });++ await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-event-filtered',+ name: tool.name,+ args: { move: 'e4', run_in_background: true },+ },+ ],+ agentId: 'agent-player',+ configurable: buildConfig([tool.name], {+ toolArguments: {+ pii: { fields: ['output'], starterPatterns: ['bearer_header'] },+ },+ }),+ metadata: { thread_id: 'event-filtered-conversation', run_id: 'event-filtered-turn' },+ });+ await flushMicrotasks();+ await flushMicrotasks();++ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(+ expect.objectContaining({+ taskId: 'event-task-filtered',+ status: 'succeeded',+ result: expect.stringContaining('content_filter_block'),+ }),+ );+ expect(JSON.stringify(eventActorDetachedAction.settle.mock.calls)).not.toContain(+ protectedValue,+ );+ expect(eventActorDetachedAction.settle).not.toHaveBeenCalledWith(+ expect.objectContaining({ status: 'failed' }),+ );+ expect(eventActorDetachedAction.wake).toHaveBeenCalledTimes(1);+ });++ it('rejects capacity before creating durable Event Actor launch authority', async () => {+ let invocations = 0;+ const tool = {+ name: 'submit_move_mcp_chess',+ description: 'submits a move',+ schema: z.object({ move: z.string() }),+ invoke: () => {+ invocations += 1;+ return new Promise(() => undefined);+ },+ } as unknown as StructuredToolInterface;+ const configurable = {+ req: {+ user: { id: 'capacity-user' },+ body: { conversationId: 'capacity-conversation' },+ },+ backgroundToolNames: [tool.name],+ };+ const metadata = { thread_id: 'capacity-conversation', run_id: 'capacity-run' };+ const ordinaryHandler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ });+ await runBatch(ordinaryHandler, {+ toolCalls: Array.from({ length: 10 }, (_, index) => ({+ id: `capacity-${index}`,+ name: tool.name,+ args: { move: 'e4', run_in_background: true },+ })),+ agentId: 'agent-player',+ configurable,+ metadata,+ });++ const reserve = jest.fn(async () => ({+ status: 'reserved' as const,+ taskId: 'event-task-capacity',+ idempotencyKey: 'f'.repeat(64),+ }));+ const detachedHandler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ eventActorDetachedAction: {+ reserve,+ markRunning: jest.fn(async () => true),+ settle: jest.fn(async () => true),+ wake: jest.fn(async () => undefined),+ },+ });++ const [rejected] = await runBatch(detachedHandler, {+ toolCalls: [+ {+ id: 'capacity-detached',+ name: tool.name,+ args: { move: 'e5', run_in_background: true },+ },+ ],+ agentId: 'agent-player',+ configurable,+ metadata,+ });++ expect(rejected.content).toContain('Too many background tasks');+ expect(invocations).toBe(10);+ expect(reserve).not.toHaveBeenCalled();+ });++ it('does not launch an expected action when staged production is disabled', async () => {+ const invoke = jest.fn(async () => ({ content: 'move accepted' }));+ const tool = {+ name: 'submit_move_mcp_chess',+ description: 'submits a move',+ schema: z.object({ move: z.string() }),+ invoke,+ } as unknown as StructuredToolInterface;+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ eventActorDetachedAction: {+ reserve: jest.fn(async () => ({+ status: 'conflict' as const,+ error: 'Detached Event Actor production is not activated',+ })),+ markRunning: jest.fn(async () => true),+ settle: jest.fn(async () => true),+ wake: jest.fn(async () => undefined),+ },+ });++ await expect(+ runBatch(handler, {+ toolCalls: [+ {+ id: 'call-disabled',+ name: tool.name,+ args: { move: 'e4', run_in_background: true },+ },+ ],+ agentId: 'agent-player',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },+ }),+ ).resolves.toEqual([+ expect.objectContaining({ content: '', errorMessage: expect.stringContaining('activated') }),+ ]);+ expect(invoke).not.toHaveBeenCalled();+ });++ it('persists a synchronous detached-tool rejection as terminal evidence', async () => {+ const tool = {+ name: 'submit_move_mcp_chess',+ description: 'submits a move',+ schema: z.object({ move: z.string() }),+ invoke: () => {+ throw new Error('launch rejected synchronously');+ },+ } as unknown as StructuredToolInterface;+ const eventActorDetachedAction = {+ reserve: jest.fn(async () => ({+ status: 'reserved' as const,+ taskId: 'event-task-sync-error',+ idempotencyKey: 'b'.repeat(64),+ })),+ markRunning: jest.fn(async () => true),+ settle: jest.fn(async () => true),+ wake: jest.fn(async () => undefined),+ };+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ eventActorDetachedAction,+ });++ await expect(+ runBatch(handler, {+ toolCalls: [+ {+ id: 'call-sync-error',+ name: tool.name,+ args: { move: 'e4', run_in_background: true },+ },+ ],+ agentId: 'agent-player',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },+ }),+ ).resolves.toEqual([+ expect.objectContaining({ content: expect.stringContaining('event-task-sync-error') }),+ ]);+ await flushMicrotasks();+ await flushMicrotasks();++ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(+ expect.objectContaining({+ taskId: 'event-task-sync-error',+ status: 'failed',+ error: expect.stringContaining('launch rejected synchronously'),+ }),+ );+ expect(eventActorDetachedAction.wake).toHaveBeenCalledTimes(1);+ });++ it('records an aborted detached expected action as cancelled', async () => {+ const abortError = Object.assign(new Error('operation aborted'), { name: 'AbortError' });+ const tool = {+ name: 'submit_move_mcp_chess',+ description: 'submits a move',+ schema: z.object({ move: z.string() }),+ invoke: async () => {+ throw abortError;+ },+ } as unknown as StructuredToolInterface;+ const eventActorDetachedAction = {+ reserve: jest.fn(async () => ({+ status: 'reserved' as const,+ taskId: 'event-task-cancelled',+ idempotencyKey: 'c'.repeat(64),+ })),+ markRunning: jest.fn(async () => true),+ settle: jest.fn(async () => true),+ wake: jest.fn(async () => undefined),+ };+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ eventActorDetachedAction,+ });++ await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-cancelled',+ name: tool.name,+ args: { move: 'e4', run_in_background: true },+ },+ ],+ agentId: 'agent-player',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'event-conversation', run_id: 'event-response' },+ });+ await flushMicrotasks();+ await flushMicrotasks();++ expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'cancelled', error: 'operation aborted' }),+ );+ });+it('returns a handle immediately, runs the tool once detached, and yields the result via check_background_task', async () => {const state = { calls: 0 } as { calls: number; lastInput?: Record<string, unknown> };const searchTool = makeSearchTool(state);diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.tsindex a87f7b2bb1..f11d891df3 100644--- a/packages/api/src/agents/handlers.ts+++ b/packages/api/src/agents/handlers.ts@@ -105,6 +105,40 @@ export interface ToolEndCallbackData {};}+export interface EventActorDetachedActionLifecycle {+ reserve(input: {+ toolName: string;+ toolCallId: string;+ turnId: string;+ arguments: unknown;+ }): Promise<+ | { status: 'ignored' }+ | { status: 'conflict'; error?: string }+ | {+ status: 'terminal';+ taskId: string;+ idempotencyKey: string;+ outcome: 'succeeded' | 'failed' | 'cancelled';+ result?: string;+ error?: string;+ }+ | {+ status: 'reserved' | 'replay';+ taskId: string;+ idempotencyKey: string;+ }+ >;+ markRunning(input: { taskId: string; idempotencyKey: string }): Promise<boolean>;+ settle(input: {+ taskId: string;+ idempotencyKey: string;+ status: 'succeeded' | 'failed' | 'cancelled';+ result?: unknown;+ error?: string;+ }): Promise<boolean>;+ wake(input: { taskId: string; idempotencyKey: string }): Promise<void>;+}+export interface ToolEndCallbackMetadata {run_id?: string;thread_id?: string;@@ -134,6 +168,8 @@ export interface ToolExecuteOptions {subagentTasks?: SubagentTaskConfig;/** Callback to process tool artifacts (code output files, file citations, etc.) */toolEndCallback?: ToolEndCallback;+ /** Durable internal-completion adapter, present only for an Event Actor invocation. */+ eventActorDetachedAction?: EventActorDetachedActionLifecycle;/*** Persists a backgrounded code-execution result onto the dispatch turn once* the detached call settles: downloads/persists generated files, patches the@@ -4232,6 +4268,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandconst {loadTools,toolEndCallback,+ eventActorDetachedAction,persistBackgroundCodeResult,emitAttachment,emitPtcProgress,@@ -4367,7 +4404,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandconst backgroundRunId = (metadata as Record<string, unknown>)?.run_id as| string| undefined;- const dispatchBackgroundToolCall = (tc: ToolCallRequest): ToolExecuteResult => {+ const dispatchBackgroundToolCall = async (+ tc: ToolCallRequest,+ ): Promise<ToolExecuteResult> => {/** A tool that failed to load must error immediately (matching the* foreground path) — a synthetic "started" handle would tell the* model a side effect is in flight that never executed. */@@ -4393,7 +4432,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandif (filtered != null) {return filtered;}- const created = backgroundTaskRegistry.create({+ const registration = {userId: backgroundUserId,conversationId: backgroundConversationId,toolCallId: tc.id,@@ -4405,8 +4444,93 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand* starts a fresh task instead of colliding. */agentId,runId: `${backgroundRunId ?? ''}:${tc.turn ?? ''}`,+ };+ const capacityAdmission =+ eventActorDetachedAction == null+ ? undefined+ : backgroundTaskRegistry.reserveCapacity(registration);+ if (capacityAdmission != null && 'atCapacity' in capacityAdmission) {+ return {+ toolCallId: tc.id,+ status: 'success' as const,+ content: buildBackgroundCapacityContent(tc.name),+ };+ }+ const capacityPermit =+ capacityAdmission != null && 'permit' in capacityAdmission+ ? capacityAdmission.permit+ : undefined;+ let detachedReservation;+ try {+ detachedReservation = await eventActorDetachedAction?.reserve({+ toolName: tc.name,+ toolCallId: tc.id,+ turnId: registration.runId,+ arguments: normalizedArgs,+ });+ } catch (error) {+ if (capacityPermit != null) {+ backgroundTaskRegistry.releaseCapacity(capacityPermit);+ }+ throw error;+ }+ if (detachedReservation?.status === 'conflict') {+ if (capacityPermit != null) {+ backgroundTaskRegistry.releaseCapacity(capacityPermit);+ }+ return {+ toolCallId: tc.id,+ status: 'error' as const,+ content: '',+ errorMessage:+ detachedReservation.error ??+ 'Detached Event Actor action conflicts with its durable launch authority',+ };+ }+ if (detachedReservation?.status === 'terminal') {+ if (capacityPermit != null) {+ backgroundTaskRegistry.releaseCapacity(capacityPermit);+ }+ if (detachedReservation.outcome === 'succeeded') {+ return {+ toolCallId: tc.id,+ status: 'success' as const,+ content: detachedReservation.result ?? '',+ };+ }+ return {+ toolCallId: tc.id,+ status: 'error' as const,+ content: '',+ errorMessage:+ detachedReservation.error ?? `Detached action ${detachedReservation.outcome}`,+ };+ }+ if (detachedReservation?.status === 'replay') {+ if (capacityPermit != null) {+ backgroundTaskRegistry.releaseCapacity(capacityPermit);+ }+ return {+ toolCallId: tc.id,+ status: 'success' as const,+ content: buildBackgroundHandleContent({+ id: detachedReservation.taskId,+ toolName: tc.name,+ status: 'running',+ }),+ };+ }+ const created = backgroundTaskRegistry.create({+ ...(detachedReservation?.status === 'reserved'+ ? { taskId: detachedReservation.taskId }+ : {}),+ ...registration,+ ...(capacityPermit == null ? {} : { capacityPermit }),});if ('atCapacity' in created) {+ if (detachedReservation?.status === 'reserved') {+ throw new Error('Detached Event Actor lost its pre-admitted background capacity');+ }return {toolCallId: tc.id,status: 'success' as const,@@ -4495,17 +4619,74 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand}})();};- void (async () => {- try {- const result = (await tool.invoke(normalizedArgs, {+ let invokePromise: Promise<{ content?: unknown; artifact?: unknown }>;+ try {+ invokePromise = Promise.resolve(+ tool.invoke(normalizedArgs, {/** Full invoke config (not just identity): a detached* code call still needs `session_id`/`_injected_files`/* `_runtime_session_hint` or it runs fileless on the* Code API's default runtime session. */toolCall: buildToolCallConfig(tc, mergedConfigurable),- configurable: mergedConfigurable,+ configurable: {+ ...mergedConfigurable,+ ...(detachedReservation?.status === 'reserved'+ ? {+ eventActorDetachedAction: {+ taskId: detachedReservation.taskId,+ idempotencyKey: detachedReservation.idempotencyKey,+ },+ }+ : {}),+ },metadata,- } as Record<string, unknown>)) as { content?: unknown; artifact?: unknown };+ } as Record<string, unknown>),+ ) as Promise<{ content?: unknown; artifact?: unknown }>;+ } catch (error) {+ /** Structured tools are permitted to reject synchronously.+ * Preserve the durable reservation and route that rejection+ * through the same terminal-evidence path as an async one. */+ invokePromise = Promise.reject(error);+ }+ const persistDetachedTerminal = async (+ input:+ | { status: 'succeeded'; result: unknown }+ | { status: 'failed' | 'cancelled'; error: string },+ ): Promise<boolean> => {+ if (+ detachedReservation?.status !== 'reserved' ||+ eventActorDetachedAction == null+ ) {+ return true;+ }+ return eventActorDetachedAction.settle({+ taskId: detachedReservation.taskId,+ idempotencyKey: detachedReservation.idempotencyKey,+ ...input,+ });+ };+ const wakeDetachedActor = async (): Promise<void> => {+ if (+ detachedReservation?.status !== 'reserved' ||+ eventActorDetachedAction == null+ ) {+ return;+ }+ try {+ await eventActorDetachedAction.wake({+ taskId: detachedReservation.taskId,+ idempotencyKey: detachedReservation.idempotencyKey,+ });+ } catch (wakeError) {+ logger.warn(+ `[event-actor] Failed to wake detached action ${detachedReservation.taskId}`,+ wakeError,+ );+ }+ };+ void (async () => {+ try {+ const result = await invokePromise;if (isCodeCall) {markCodeSandboxWarm();}@@ -4523,6 +4704,14 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandconst errorOutput = isCodeCall? toCodeToolFailure(tc.name, policyError): policyError;+ if (+ !(await persistDetachedTerminal({+ status: 'succeeded',+ result: errorOutput,+ }))+ ) {+ return;+ }backgroundTaskRegistry.fail(backgroundUserId,backgroundConversationId,@@ -4531,6 +4720,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand{ harvestStarted: harvestEnabled },);harvestCodeResult({ output: errorOutput });+ await wakeDetachedActor();return;}/** Hold any artifact (images, files, UI resources,@@ -4540,6 +4730,14 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand* artifactPromises are already awaited and the stream is* closed, so that push would be silently dropped. The poll* turn delivers it live in `check_background_task`. */+ if (+ !(await persistDetachedTerminal({+ status: 'succeeded',+ result: content,+ }))+ ) {+ return;+ }backgroundTaskRegistry.complete(backgroundUserId,backgroundConversationId,@@ -4550,6 +4748,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandoutput: typeof content === 'string' ? content : undefined,artifact: result.artifact,});+ await wakeDetachedActor();} catch (toolError) {const policyError =toolError instanceof ContentFilterError@@ -4569,6 +4768,20 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandisCodeCall && (policyError != null || filteredError != null)? toCodeToolFailure(tc.name, neutralizedError): neutralizedError;+ const detachedTerminalStatus =+ toolError instanceof Error &&+ (toolError.name === 'AbortError' ||+ (toolError as Error & { code?: string }).code === 'ABORT_ERR')+ ? 'cancelled'+ : 'failed';+ if (+ !(await persistDetachedTerminal({+ status: detachedTerminalStatus,+ error: deliveredError,+ }))+ ) {+ return;+ }backgroundTaskRegistry.fail(backgroundUserId,backgroundConversationId,@@ -4580,8 +4793,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand{ harvestStarted: harvestEnabled },);harvestCodeResult({ output: deliveredError });+ await wakeDetachedActor();}})();+ if (+ detachedReservation?.status === 'reserved' &&+ eventActorDetachedAction != null &&+ !(await eventActorDetachedAction.markRunning({+ taskId: detachedReservation.taskId,+ idempotencyKey: detachedReservation.idempotencyKey,+ }))+ ) {+ throw new Error('Detached Event Actor launch acknowledgement is stale');+ }}return {toolCallId: tc.id,@@ -4836,7 +5060,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandpersistBackgroundCodeResult == null)) {- return reportResult(dispatchBackgroundToolCall(tc));+ return reportResult(await dispatchBackgroundToolCall(tc));}const execute = async (diff --git a/packages/api/src/agents/plan.spec.ts b/packages/api/src/agents/plan.spec.tsindex e98317ff26..dff85c32aa 100644--- a/packages/api/src/agents/plan.spec.ts+++ b/packages/api/src/agents/plan.spec.ts@@ -7,7 +7,6 @@ const baseInput = (): ResolveAgentTurnExecutionPlanInput => ({isNewConversation: false,canPause: false,durableEventActorSuspensions: false,- expectedActionMayDetach: false,});const boundEvent = () => ({@@ -67,7 +66,6 @@ describe('resolveAgentTurnExecutionPlan', () => {['no binding', { event: { ...boundEvent(), binding: undefined } }],['no expected action', { event: { ...boundEvent(), expectedAction: undefined } }],['memory checkpointer', { event: boundEvent(), checkpointerType: 'memory' }],- ['detachable action', { event: boundEvent(), expectedActionMayDetach: true }],['pre-cutover pause consumer fleet',{ event: boundEvent(), canPause: true, durableEventActorSuspensions: false },diff --git a/packages/api/src/agents/plan.ts b/packages/api/src/agents/plan.tsindex c15c88d69f..ff47d60c4d 100644--- a/packages/api/src/agents/plan.ts+++ b/packages/api/src/agents/plan.ts@@ -38,7 +38,6 @@ export interface ResolveAgentTurnExecutionPlanInput {* after every resume consumer understands signed Event Actor suspensions. */durableEventActorSuspensions: boolean;checkpointerType?: TCheckpointerType;- expectedActionMayDetach: boolean;}const COMPLETION_EVENT_TYPE = 'subagent.completion';@@ -73,7 +72,6 @@ export function resolveAgentTurnExecutionPlan(binding != null &&expectedAction != null &&input.checkpointerType !== 'memory' &&- !input.expectedActionMayDetach &&(!input.canPause || input.durableEventActorSuspensions);let strategy: AgentTurnContinuationStrategy = 'history';if (input.isNewConversation) {diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.mdindex 55e60a9f1b..b18f4108ce 100644--- a/packages/api/src/agents/triggers/README.md+++ b/packages/api/src/agents/triggers/README.md@@ -127,6 +127,17 @@ Bound child continuations are automatic after authentication and binding authori`endpoints.agents.eventDriven.selfUrl`; most deployments should omit both and use the boundlistener.+Detached Event Actor completion production has an explicit rolling-deployment fence. It remains+fail-closed until Redis generation streams are active and+`AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED=true` is set. Deploy the new version to every+replica first, verify that no old API or worker replica remains, and only then enable the producer+across the Redis-backed fleet. This ordering ensures Redis recovery, HTTP completion consumers,+account-deletion drains, lane maintenance, and operational requeue/dead-letter paths all understand+capability-fenced completion work before any is created. For rollback, disable the producer, drain+all detached completion work, and only then start an older version. Internal detached completions+always target the current replica's bound listener; `AGENT_TRIGGERS_SELF_URL` remains available for+ordinary trigger dispatch but cannot route capability-owned completion work to another replica.+```httpPOST /api/agents/v1/events/bindingsAuthorization: Bearer <remote-agents-api-key>diff --git a/packages/api/src/agents/triggers/actor.spec.ts b/packages/api/src/agents/triggers/actor.spec.tsindex 4a429ff61d..64cc5d6998 100644--- a/packages/api/src/agents/triggers/actor.spec.ts+++ b/packages/api/src/agents/triggers/actor.spec.ts@@ -495,6 +495,114 @@ describe('event actor host adapter', () => {expect(dependencies.commitState).not.toHaveBeenCalled();});+ it('carries applied expected-action evidence across a later re-pause', async () => {+ let storedSuspension: IAgentEventActorSuspension | undefined;+ let pendingPause:+ | {+ actionId: string;+ jobCreatedAt: number;+ interrupt: EventActorInterrupt;+ }+ | undefined;+ let observedAction: { toolName: string; toolCallId?: string } | undefined;+ const dependencies = {+ ...deps(),+ storeSuspension: jest.fn(async (input) => {+ storedSuspension = {+ suspension: input.suspension,+ actionId: input.actionId,+ jobCreatedAt: input.jobCreatedAt,+ appliedAction: input.appliedAction,+ status: 'pending',+ observedAt: new Date(),+ };+ return { status: 'stored' as const };+ }),+ claimSuspension: jest.fn(async ({ resumeAttemptId }) => {+ if (storedSuspension == null) {+ throw new Error('test suspension was not stored');+ }+ storedSuspension = { ...storedSuspension, status: 'claimed', resumeAttemptId };+ return { status: 'claimed' as const };+ }),+ settleSuspension: jest.fn(async () => ({ status: 'settled' as const })),+ };+ dependencies.getSnapshot.mockImplementation(async () => ({+ state,+ reconciliations: [],+ legacyTurn: null,+ suspension: storedSuspension ?? null,+ epoch,+ }));+ const initial = await executeAgentEventActor(+ {+ user: 'user-1',+ conversationId,+ invocationId: 'event-action-repause',+ event: { id: 'event-action-repause' },+ signal: new AbortController().signal,+ invoke: async () => 'initial-pause',+ readAppliedAction: () => undefined,+ readSuspension: () => ({+ actionId: 'detached-task',+ jobCreatedAt: 801,+ interrupt: { id: 'detached-task', payload: { type: 'detached' } },+ }),+ },+ dependencies,+ );+ if (initial.execution.status !== 'suspended') {+ throw new Error('test setup did not suspend');+ }+ observedAction = { toolName: 'submit_move', toolCallId: 'call-detached' };+ pendingPause = {+ actionId: 'ask-user',+ jobCreatedAt: 802,+ interrupt: { id: 'ask-user', payload: { type: 'ask_user_question' } },+ };+ const repaused = await resumeAgentEventActor(+ {+ user: 'user-1',+ conversationId,+ suspension: initial.execution.suspension,+ resumeAttemptId: 'resume-detached',+ resumeValue: { status: 'succeeded' },+ signal: new AbortController().signal,+ resume: async () => 'asks-user',+ readAppliedAction: () => observedAction,+ readSuspension: () => pendingPause,+ },+ dependencies,+ );+ expect(repaused.execution.status).toBe('suspended');+ expect(dependencies.storeSuspension).toHaveBeenLastCalledWith(+ expect.objectContaining({ appliedAction: observedAction }),+ );+ if (repaused.execution.status !== 'suspended') {+ throw new Error('test setup did not re-pause');+ }+ observedAction = undefined;+ pendingPause = undefined;+ const completed = await resumeAgentEventActor(+ {+ user: 'user-1',+ conversationId,+ suspension: repaused.execution.suspension,+ resumeAttemptId: 'resume-human',+ resumeValue: { answer: 'continue' },+ signal: new AbortController().signal,+ resume: async () => 'completed',+ readAppliedAction: () => observedAction,+ readSuspension: () => pendingPause,+ },+ dependencies,+ );+ expect(completed.execution).toMatchObject({+ status: 'applied',+ result: { action: { toolName: 'submit_move', toolCallId: 'call-detached' } },+ });+ });+it('cold-starts once, then forks and warm-continues only the next event', async () => {const dependencies = deps();const invocations: Array<{ continuation: string; checkpointId?: string }> = [];diff --git a/packages/api/src/agents/triggers/actor.ts b/packages/api/src/agents/triggers/actor.tsindex 985fc0ca8c..ec5bdb39ac 100644--- a/packages/api/src/agents/triggers/actor.ts+++ b/packages/api/src/agents/triggers/actor.ts@@ -64,6 +64,7 @@ export interface ExecuteAgentEventActorInput<T> {readAppliedAction(): AgentEventAppliedAction | undefined;readSuspension?():| {+ kind?: 'human_decision' | 'internal_completion';actionId: string;jobCreatedAt: number;interrupt: EventActorInterrupt;@@ -118,6 +119,7 @@ export interface ResumeAgentEventActorInput<T> {readAppliedAction(): AgentEventAppliedAction | undefined;readSuspension?():| {+ kind?: 'human_decision' | 'internal_completion';actionId: string;jobCreatedAt: number;interrupt: EventActorInterrupt;@@ -237,7 +239,14 @@ export async function executeAgentEventActor<T>(let preparedContext: AgentEventActorContext | undefined;let resultContext: AgentEventActorContext | undefined;let pendingSuspension:- | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt }+ | {+ kind?: 'human_decision' | 'internal_completion';+ appliedAction?: AgentEventAppliedAction;+ handlingGenerationCreatedAt?: number;+ actionId: string;+ jobCreatedAt: number;+ interrupt: EventActorInterrupt;+ }| undefined;let actionAppliedBeforePause = false;const adapter: EventActorHostAdapter<EventActorEvent, EventActorResult> = {@@ -617,6 +626,10 @@ export async function executeAgentEventActor<T>(conversationId: input.conversationId,...(input.tenantId == null ? {} : { tenantId: input.tenantId }),suspension: request.suspension as IAgentEventActorSuspensionEvidence,+ ...(pendingSuspension.kind == null ? {} : { kind: pendingSuspension.kind }),+ ...(pendingSuspension.handlingGenerationCreatedAt == null+ ? {}+ : { handlingGenerationCreatedAt: pendingSuspension.handlingGenerationCreatedAt }),actionId: pendingSuspension.actionId,jobCreatedAt: pendingSuspension.jobCreatedAt,...(actionAppliedBeforePause ? { invalidateHead: true } : {}),@@ -900,7 +913,14 @@ export async function resumeAgentEventActor<T>(let observedEpoch: number | undefined;let resultContext: AgentEventActorContext | undefined;let pendingSuspension:- | { actionId: string; jobCreatedAt: number; interrupt: EventActorInterrupt }+ | {+ kind?: 'human_decision' | 'internal_completion';+ appliedAction?: AgentEventAppliedAction;+ handlingGenerationCreatedAt?: number;+ actionId: string;+ jobCreatedAt: number;+ interrupt: EventActorInterrupt;+ }| undefined;let actionAppliedBeforePause = false;@@ -979,9 +999,17 @@ export async function resumeAgentEventActor<T>(* human boundary. Publish the successor suspension before considering* the segment terminal; otherwise the successor checkpoint is committed* without any resumable host action. */- pendingSuspension = input.readSuspension?.();- if (pendingSuspension != null) {- actionAppliedBeforePause = input.readAppliedAction() != null;+ const observedAction = input.readAppliedAction() ?? hostSuspension.appliedAction;+ const action = observedAction == null ? undefined : toEventActorAppliedAction(observedAction);+ const observedSuspension = input.readSuspension?.();+ if (observedSuspension != null) {+ pendingSuspension = {+ ...observedSuspension,+ ...(observedAction == null ? {} : { appliedAction: observedAction }),+ handlingGenerationCreatedAt:+ hostSuspension.handlingGenerationCreatedAt ?? hostSuspension.jobCreatedAt,+ };+ actionAppliedBeforePause = observedAction != null;const checkpoint = await captureAgentEventCheckpoint(input.conversationId,request.suspension.checkpoint.checkpointNs,@@ -1006,8 +1034,6 @@ export async function resumeAgentEventActor<T>(},};}- const observedAction = input.readAppliedAction();- const action = observedAction == null ? undefined : toEventActorAppliedAction(observedAction);if (action == null) {if (invocationError != null) {return { status: 'claimed_failed', error: asError(invocationError) };@@ -1079,6 +1105,13 @@ export async function resumeAgentEventActor<T>(return deps.storeSuspension({...owner,suspension: request.suspension as IAgentEventActorSuspensionEvidence,+ ...(pendingSuspension.kind == null ? {} : { kind: pendingSuspension.kind }),+ ...(pendingSuspension.appliedAction == null+ ? {}+ : { appliedAction: pendingSuspension.appliedAction }),+ ...(pendingSuspension.handlingGenerationCreatedAt == null+ ? {}+ : { handlingGenerationCreatedAt: pendingSuspension.handlingGenerationCreatedAt }),actionId: pendingSuspension.actionId,jobCreatedAt: pendingSuspension.jobCreatedAt,...(actionAppliedBeforePause ? { invalidateHead: true } : {}),diff --git a/packages/api/src/agents/triggers/bindingResolver.spec.ts b/packages/api/src/agents/triggers/bindingResolver.spec.tsindex 9e21bd1bfe..5ec80fb47c 100644--- a/packages/api/src/agents/triggers/bindingResolver.spec.ts+++ b/packages/api/src/agents/triggers/bindingResolver.spec.ts@@ -1,5 +1,9 @@import { AIMessage } from '@langchain/core/messages';import type { IConversation } from '@librechat/data-schemas';+import {+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+} from './detachedAction';import { createAgentTriggerEnvelope, type AgentContinueTriggerEnvelope } from './envelope';import { createAgentEventContinueResolver } from './bindingResolver';@@ -30,6 +34,62 @@ function envelope(): AgentContinueTriggerEnvelope {}) as AgentContinueTriggerEnvelope;}+function detachedCompletionEnvelope(generationCreatedAt: number): AgentContinueTriggerEnvelope {+ return createAgentTriggerEnvelope({+ mode: 'continue',+ requestId: 'request-completion',+ deliveryId: 'detached_completion:task-1',+ receivedAt: 2,+ principal: { id: 'user-1', tenantId: 'tenant-1' },+ event: {+ id: 'task-1',+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ occurredAt: 2,+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },+ payload: {+ version: 1,+ invocationId: 'delivery-1',+ generationCreatedAt,+ wakeGenerationCreatedAt: generationCreatedAt,+ taskId: 'task-1',+ idempotencyKey: 'a'.repeat(64),+ },+ },+ input: 'Detached completion.',+ target: {+ agentId: 'agent-player',+ conversationId: 'child-thread',+ parentMessageId: 'placeholder',+ bindingId,+ sourceKeyId,+ },+ }) as AgentContinueTriggerEnvelope;+}++function boundMethods() {+ return {+ getAgentEventBinding: jest.fn(async () => ({+ conversationId: 'child-thread',+ agentId: 'agent-player',+ tenantId: 'tenant-1',+ binding: { bindingId, sourceKeyId, actorId: 'player' },+ lineage: {+ parentConversationId: 'parent-thread',+ parentAgentId: 'agent-director',+ } as never,+ })),+ getConvo: jest.fn(+ async () =>+ ({+ conversationId: 'parent-thread',+ agent_id: 'agent-director',+ tenantId: 'tenant-1',+ }) as IConversation,+ ),+ getMessages: jest.fn(async () => []) as never,+ };+}+describe('agent event continuation resolver', () => {it('re-resolves the latest assistant leaf immediately before dispatch', async () => {const getMessages = jest.fn(async () => [@@ -129,6 +189,33 @@ describe('agent event continuation resolver', () => {});});+ it('admits only the exact detached completion through its terminal host-action fence', async () => {+ const createdAt = 77;+ const resolver = createAgentEventContinueResolver({+ getGenerationJob: jest.fn(async () => ({+ status: 'complete',+ createdAt,+ metadata: { terminalPersistencePending: true },+ })),+ methods: boundMethods(),+ });++ await expect(+ resolver(detachedCompletionEnvelope(createdAt), {+ idempotencyKey: 'completion-1',+ } as never),+ ).resolves.toMatchObject({ status: 'ready' });++ await expect(+ resolver(detachedCompletionEnvelope(createdAt + 1), {+ idempotencyKey: 'completion-stale',+ } as never),+ ).rejects.toMatchObject({+ code: 'EVENT_ACTOR_NOT_READY',+ deferWithoutAttempt: true,+ });+ });+it('fails closed after the binding parent is removed', async () => {const resolver = createAgentEventContinueResolver({methods: {diff --git a/packages/api/src/agents/triggers/bindingResolver.ts b/packages/api/src/agents/triggers/bindingResolver.tsindex 02c9e26483..cfa964abf0 100644--- a/packages/api/src/agents/triggers/bindingResolver.ts+++ b/packages/api/src/agents/triggers/bindingResolver.ts@@ -3,6 +3,11 @@ import type { ConversationMethods, MessageMethods } from '@librechat/data-schemaimport type { AgentTriggerContinuePreparation, AgentTriggerExecutionHostDeps } from './host';import type { AgentContinueTriggerEnvelope } from './envelope';import type { AgentTriggerDispatchContext } from './dispatch';+import {+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ parseAgentEventActorDetachedCompletion,+} from './detachedAction';import { isAgentEventRetentionActive } from '../eventRetention';import { AgentTriggerExecutionError } from './host';@@ -14,6 +19,7 @@ export interface AgentEventContinueResolverDeps {getGenerationJob?: (conversationId: string) => Promise<| {status?: string;+ createdAt?: number;metadata?: { terminalPersistencePending?: boolean };}| null@@ -108,10 +114,19 @@ export function createAgentEventContinueResolver({true,);}+ const detachedCompletion =+ envelope.event.type === EVENT_ACTOR_DETACHED_COMPLETION_TYPE &&+ envelope.event.source.type === 'internal' &&+ envelope.event.source.id === EVENT_ACTOR_DETACHED_COMPLETION_SOURCE+ ? parseAgentEventActorDetachedCompletion(envelope.event.payload)+ : undefined;+ const ownsTerminalWake =+ detachedCompletion != null &&+ active?.createdAt === detachedCompletion.wakeGenerationCreatedAt;if (active?.status === 'running' ||active?.status === 'requires_action' ||- active?.metadata?.terminalPersistencePending === true+ (active?.metadata?.terminalPersistencePending === true && !ownsTerminalWake)) {throw new AgentTriggerExecutionError('The event actor is still handling an earlier turn.', {mode: 'continue',diff --git a/packages/api/src/agents/triggers/delivery.ts b/packages/api/src/agents/triggers/delivery.tsindex 33c0e2d345..4920033a47 100644--- a/packages/api/src/agents/triggers/delivery.ts+++ b/packages/api/src/agents/triggers/delivery.ts@@ -21,6 +21,9 @@ export interface AgentTriggerEnqueueOptions {/** Opt-in only for observational bound-child continuations. Actionable,* fenced, approval, HITL, and control deliveries must remain individual. */coalesce?: AgentTriggerCoalesceOptions;+ /** Server-owned capability fence. Deliveries carrying this marker remain+ * invisible to older workers during a rolling deployment. */+ requiredWorkerCapability?: string;}export interface PreparedAgentTriggerDelivery {@@ -38,6 +41,7 @@ export interface PreparedAgentTriggerDelivery {/** Persisted rollout marker: keep this bound actor lane queued until the* admitted child turn records an authoritative terminal outcome. */awaitTerminalHandling?: boolean;+ requiredWorkerCapability?: string;}export class AgentTriggerDeliveryError extends TypeError {@@ -188,6 +192,9 @@ export function prepareAgentTriggerDelivery(...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),availableAt: coalesceUntil ?? requestedAvailableAt,envelopeBytes: bytes,+ ...(options.requiredWorkerCapability == null+ ? {}+ : { requiredWorkerCapability: options.requiredWorkerCapability }),...(coalesceKey != null &&coalesceUntil != null && {coalesceKey,diff --git a/packages/api/src/agents/triggers/detachedAction.spec.ts b/packages/api/src/agents/triggers/detachedAction.spec.tsnew file mode 100644index 0000000000..c5df8e949a--- /dev/null+++ b/packages/api/src/agents/triggers/detachedAction.spec.ts@@ -0,0 +1,551 @@+import type { AgentTriggerDeliveryRecord } from '@librechat/data-schemas';+import type { AgentTriggerEnvelope } from './envelope';+import {+ createAgentEventActorDetachedActionLifecycle,+ createAgentEventDetachedResumeHandler,+ isAgentEventActorDetachedActionProducerEnabled,+} from './detachedAction';++describe('isAgentEventActorDetachedActionProducerEnabled', () => {+ const key = 'AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED';+ const original = process.env[key];++ afterEach(() => {+ if (original == null) {+ delete process.env[key];+ return;+ }+ process.env[key] = original;+ });++ it('requires an explicit true activation', () => {+ delete process.env[key];+ expect(isAgentEventActorDetachedActionProducerEnabled()).toBe(false);+ process.env[key] = 'false';+ expect(isAgentEventActorDetachedActionProducerEnabled()).toBe(false);+ process.env[key] = ' TRUE ';+ expect(isAgentEventActorDetachedActionProducerEnabled()).toBe(true);+ });+});++describe('createAgentEventActorDetachedActionLifecycle', () => {+ it('owns only the exact expected action and exposes a suspension after launch', async () => {+ const action = {+ version: 1 as const,+ invocationId: 'delivery-1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-1',+ turnId: 'response-1:0',+ taskId: `event_actor_${'a'.repeat(64)}`,+ idempotencyKey: 'a'.repeat(64),+ launchAttempt: 0 as const,+ status: 'reserved' as const,+ reservedAt: new Date('2026-08-28T12:00:00.000Z'),+ observedAt: new Date('2026-08-28T12:00:00.000Z'),+ recoveryAfter: new Date('2026-08-28T12:01:00.000Z'),+ };+ const reserve = jest.fn(async () => ({ status: 'reserved' as const, action }));+ const markRunning = jest.fn(async () => ({ status: 'applied' as const }));+ const settle = jest+ .fn()+ .mockRejectedValueOnce(new Error('mongo unavailable'))+ .mockResolvedValue({ status: 'applied' as const });+ const waitForTerminalPersistenceRetry = jest.fn(async () => undefined);+ const persistTerminalEvidence = jest.fn(async () => undefined);+ const wake = jest.fn(async () => undefined);+ const lifecycle = createAgentEventActorDetachedActionLifecycle(+ {+ user: 'user-1',+ tenantId: 'tenant-1',+ bindingId: 'binding-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 123,+ turnCreatedAt: 456,+ invocationId: 'delivery-1',+ expectedAction: {+ toolName: 'submit_move',+ argumentSubset: { gameId: 'game-1' },+ },+ },+ {+ reserveAgentEventActorDetachedAction: reserve,+ markAgentEventActorDetachedActionRunning: markRunning,+ settleAgentEventActorDetachedAction: settle,+ persistTerminalEvidence,+ onTerminal: wake,+ waitForTerminalPersistenceRetry,+ producerEnabled: () => true,+ now: () => new Date('2026-08-28T12:00:00.000Z'),+ },+ );++ await expect(+ lifecycle.reserve({+ toolName: 'unrelated_tool',+ toolCallId: 'call-0',+ turnId: 'response-1:0',+ arguments: { gameId: 'game-1' },+ }),+ ).resolves.toEqual({ status: 'ignored' });+ await expect(+ lifecycle.reserve({+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-1',+ turnId: 'response-1:0',+ arguments: { gameId: 'game-1', move: 'e4' },+ }),+ ).resolves.toEqual({+ status: 'reserved',+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ });+ expect(lifecycle.readSuspension()).toBeUndefined();++ await expect(+ lifecycle.markRunning({ taskId: action.taskId, idempotencyKey: action.idempotencyKey }),+ ).resolves.toBe(true);++ expect(lifecycle.readSuspension()).toEqual({+ kind: 'internal_completion',+ actionId: action.taskId,+ jobCreatedAt: 456,+ interrupt: {+ id: action.taskId,+ payload: {+ type: 'event_actor_detached_action',+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ },+ },+ });+ await expect(+ lifecycle.settle({+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ status: 'succeeded',+ result: { content: 'move accepted' },+ }),+ ).resolves.toBe(true);+ await lifecycle.wake({ taskId: action.taskId, idempotencyKey: action.idempotencyKey });+ expect(settle).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'succeeded', result: '{"content":"move accepted"}' }),+ );+ expect(settle).toHaveBeenCalledTimes(2);+ expect(persistTerminalEvidence).toHaveBeenCalledTimes(1);+ expect(persistTerminalEvidence).toHaveBeenCalledWith({+ version: 1,+ deliveryKey: 'delivery-1',+ generationCreatedAt: 123,+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ status: 'succeeded',+ result: '{"content":"move accepted"}',+ observedAt: new Date('2026-08-28T12:00:00.000Z').getTime(),+ });+ expect(waitForTerminalPersistenceRetry).toHaveBeenCalledWith(100);+ expect(reserve).toHaveBeenCalledWith(expect.objectContaining({ turnId: 'response-1:0' }));+ expect(wake).toHaveBeenCalledWith({+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ });+ });++ it('fails closed before durable reservation until detached production is activated', async () => {+ const reserve = jest.fn();+ const lifecycle = createAgentEventActorDetachedActionLifecycle(+ {+ user: 'user-1',+ bindingId: 'binding-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 123,+ turnCreatedAt: 123,+ invocationId: 'delivery-1',+ expectedAction: { toolName: 'submit_move' },+ },+ {+ reserveAgentEventActorDetachedAction: reserve,+ markAgentEventActorDetachedActionRunning: jest.fn(),+ settleAgentEventActorDetachedAction: jest.fn(),+ persistTerminalEvidence: jest.fn(),+ onTerminal: jest.fn(),+ producerEnabled: () => false,+ },+ );++ await expect(+ lifecycle.reserve({+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-1',+ turnId: 'response-1:0',+ arguments: {},+ }),+ ).resolves.toEqual({+ status: 'conflict',+ error: expect.stringContaining('not activated'),+ });+ expect(reserve).not.toHaveBeenCalled();+ expect(lifecycle.readSuspension()).toBeUndefined();+ });++ it('returns exact terminal evidence instead of projecting a stale running handle', async () => {+ const action = {+ version: 1 as const,+ invocationId: 'delivery-1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-terminal',+ turnId: 'response-1:0',+ taskId: `event_actor_${'b'.repeat(64)}`,+ idempotencyKey: 'b'.repeat(64),+ launchAttempt: 1,+ status: 'failed' as const,+ reservedAt: new Date(),+ observedAt: new Date(),+ settledAt: new Date(),+ recoveryAfter: new Date(),+ error: 'service unavailable',+ };+ const lifecycle = createAgentEventActorDetachedActionLifecycle(+ {+ user: 'user-1',+ bindingId: 'binding-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 123,+ turnCreatedAt: 123,+ invocationId: 'delivery-1',+ expectedAction: { toolName: 'submit_move' },+ },+ {+ reserveAgentEventActorDetachedAction: jest.fn(async () => ({+ status: 'replay' as const,+ action,+ })),+ markAgentEventActorDetachedActionRunning: jest.fn(),+ settleAgentEventActorDetachedAction: jest.fn(),+ persistTerminalEvidence: jest.fn(),+ onTerminal: jest.fn(),+ producerEnabled: () => true,+ },+ );++ await expect(+ lifecycle.reserve({+ toolName: action.toolName,+ toolCallId: action.toolCallId,+ turnId: 'response-1:0',+ arguments: {},+ }),+ ).resolves.toEqual({+ status: 'terminal',+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ outcome: 'failed',+ error: action.error,+ });+ expect(lifecycle.readSuspension()).toBeUndefined();+ });++ it('stages terminal evidence durably before retrying the authoritative write', async () => {+ const action = {+ version: 1 as const,+ invocationId: 'delivery-1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-outbox',+ turnId: 'response-1:0',+ taskId: `event_actor_${'e'.repeat(64)}`,+ idempotencyKey: 'e'.repeat(64),+ launchAttempt: 0,+ status: 'reserved' as const,+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ };+ const persistTerminalEvidence = jest+ .fn()+ .mockRejectedValueOnce(new Error('job store unavailable'))+ .mockResolvedValue(undefined);+ const settle = jest.fn().mockResolvedValue({ status: 'applied' });+ const waitForTerminalPersistenceRetry = jest.fn(async () => undefined);+ const lifecycle = createAgentEventActorDetachedActionLifecycle(+ {+ user: 'user-1',+ bindingId: 'binding-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 123,+ turnCreatedAt: 123,+ invocationId: 'delivery-1',+ expectedAction: { toolName: 'submit_move' },+ },+ {+ reserveAgentEventActorDetachedAction: jest.fn(async () => ({+ status: 'reserved' as const,+ action,+ })),+ markAgentEventActorDetachedActionRunning: jest.fn(async () => ({+ status: 'applied' as const,+ })),+ settleAgentEventActorDetachedAction: settle,+ persistTerminalEvidence,+ onTerminal: jest.fn(),+ waitForTerminalPersistenceRetry,+ producerEnabled: () => true,+ },+ );+ const reservation = await lifecycle.reserve({+ toolName: action.toolName,+ toolCallId: action.toolCallId,+ turnId: 'response-1:0',+ arguments: {},+ });+ expect(reservation.status).toBe('reserved');++ await expect(+ lifecycle.settle({+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ status: 'failed',+ error: 'launch failed',+ }),+ ).resolves.toBe(true);++ expect(persistTerminalEvidence).toHaveBeenCalledTimes(2);+ expect(settle).toHaveBeenCalledTimes(1);+ expect(persistTerminalEvidence.mock.invocationCallOrder[1]).toBeLessThan(+ settle.mock.invocationCallOrder[0],+ );+ expect(waitForTerminalPersistenceRetry).toHaveBeenCalledWith(100);+ });++ it('preserves a same-executor replay of its running reservation', async () => {+ const action = {+ version: 1 as const,+ invocationId: 'delivery-1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-replay',+ turnId: 'response-1:0',+ taskId: `event_actor_${'d'.repeat(64)}`,+ idempotencyKey: 'd'.repeat(64),+ launchAttempt: 0,+ status: 'reserved' as const,+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ };+ const reserve = jest+ .fn()+ .mockResolvedValueOnce({ status: 'reserved' as const, action })+ .mockResolvedValueOnce({+ status: 'replay' as const,+ action: { ...action, status: 'running' as const },+ });+ const lifecycle = createAgentEventActorDetachedActionLifecycle(+ {+ user: 'user-1',+ bindingId: 'binding-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 123,+ turnCreatedAt: 123,+ invocationId: 'delivery-1',+ expectedAction: { toolName: 'submit_move' },+ },+ {+ reserveAgentEventActorDetachedAction: reserve,+ markAgentEventActorDetachedActionRunning: jest.fn(),+ settleAgentEventActorDetachedAction: jest.fn(),+ persistTerminalEvidence: jest.fn(),+ onTerminal: jest.fn(),+ producerEnabled: () => true,+ },+ );+ const input = {+ toolName: action.toolName,+ toolCallId: action.toolCallId,+ turnId: 'response-1:0',+ arguments: {},+ };++ await expect(lifecycle.reserve(input)).resolves.toEqual({+ status: 'reserved',+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ });+ await expect(lifecycle.reserve(input)).resolves.toEqual({+ status: 'replay',+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ });+ });++ it('refuses an indeterminate replay', async () => {+ const baseAction = {+ version: 1 as const,+ invocationId: 'delivery-1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-release',+ turnId: 'response-1:0',+ taskId: `event_actor_${'c'.repeat(64)}`,+ idempotencyKey: 'c'.repeat(64),+ launchAttempt: 0,+ status: 'reserved' as const,+ reservedAt: new Date('2026-08-28T12:00:00.000Z'),+ observedAt: new Date('2026-08-28T12:00:00.000Z'),+ recoveryAfter: new Date('2026-08-28T12:01:00.000Z'),+ };+ const reserve = jest+ .fn()+ .mockResolvedValueOnce({ status: 'reserved' as const, action: baseAction })+ .mockResolvedValueOnce({+ status: 'replay' as const,+ action: { ...baseAction, status: 'launch_indeterminate' as const },+ });+ const dependencies = {+ reserveAgentEventActorDetachedAction: reserve,+ markAgentEventActorDetachedActionRunning: jest.fn(),+ settleAgentEventActorDetachedAction: jest.fn(),+ persistTerminalEvidence: jest.fn(),+ onTerminal: jest.fn(),+ producerEnabled: () => true,+ now: () => new Date('2026-08-28T12:00:00.000Z'),+ };+ const owner = {+ user: 'user-1',+ bindingId: 'binding-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 123,+ turnCreatedAt: 123,+ invocationId: 'delivery-1',+ expectedAction: { toolName: 'submit_move' },+ };+ const first = createAgentEventActorDetachedActionLifecycle(owner, dependencies);+ const reservation = await first.reserve({+ toolName: baseAction.toolName,+ toolCallId: baseAction.toolCallId,+ turnId: 'response-1:0',+ arguments: {},+ });+ expect(reservation.status).toBe('reserved');+ expect(first.readSuspension()).toBeUndefined();++ const recovered = createAgentEventActorDetachedActionLifecycle(owner, dependencies);+ await expect(+ recovered.reserve({+ toolName: baseAction.toolName,+ toolCallId: baseAction.toolCallId,+ turnId: 'response-1:0',+ arguments: {},+ }),+ ).resolves.toEqual(+ expect.objectContaining({+ status: 'conflict',+ error: expect.stringContaining('indeterminate'),+ }),+ );+ });+});++describe('createAgentEventDetachedResumeHandler', () => {+ it('enqueues duplicate completion wakes under one stable mailbox identity', async () => {+ const envelope = {+ version: 1 as const,+ mode: 'continue' as const,+ requestId: 'request-original',+ deliveryId: 'delivery-original',+ receivedAt: 1,+ principal: { userId: 'user-1' },+ event: {+ id: 'event-1',+ type: 'game.move',+ occurredAt: 1,+ source: { id: 'source-1', type: 'remote_api_key' },+ },+ target: {+ agentId: 'agent-1',+ conversationId: 'conversation-1',+ parentMessageId: 'message-1',+ bindingId: 'binding-1',+ sourceKeyId: 'source-key-1',+ },+ input: 'play',+ expectedAction: { toolName: 'submit_move' },+ };+ const delivery = {+ id: 'row-1',+ deliveryKey: 'trigger_original',+ fingerprint: 'fingerprint',+ orderingKey: 'lane',+ laneSequence: 1,+ envelope,+ user: 'user-1',+ status: 'succeeded',+ attempts: 1,+ availableAt: new Date(),+ createdAt: new Date(),+ } as unknown as AgentTriggerDeliveryRecord;+ const enqueueAgentTrigger = jest.fn(async (_envelope: AgentTriggerEnvelope) => undefined);+ let request = 0;+ const resume = createAgentEventDetachedResumeHandler({+ getAgentTriggerDelivery: jest.fn(async () => delivery),+ enqueueAgentTrigger,+ requestId: () => `wake-${++request}`,+ now: () => 1_787_000_001_000,+ });+ const action = {+ version: 1 as const,+ invocationId: 'trigger_original',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-1',+ turnId: 'response-1:0',+ taskId: 'task-1',+ idempotencyKey: 'a'.repeat(64),+ launchAttempt: 0,+ status: 'succeeded' as const,+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ settledAt: new Date(1_787_000_000_500),+ result: 'accepted',+ };+ const input = {+ streamId: 'conversation-1',+ handlingGenerationCreatedAt: 1_786_999_999_000,+ job: {+ streamId: 'conversation-1',+ conversationId: 'conversation-1',+ userId: 'user-1',+ status: 'complete' as const,+ createdAt: 1_787_000_000_000,+ syncSent: false,+ agentEventDeliveryKey: 'trigger_original',+ agentEventBindingId: 'binding-1',+ },+ suspension: {} as never,+ action,+ };++ await resume(input);+ await resume(input);++ expect(enqueueAgentTrigger).toHaveBeenCalledTimes(2);+ expect(enqueueAgentTrigger).toHaveBeenNthCalledWith(1, expect.any(Object), {+ requiredWorkerCapability: 'event_actor_detached_action_v1',+ });+ const [first] = enqueueAgentTrigger.mock.calls[0];+ const [second] = enqueueAgentTrigger.mock.calls[1];+ expect(first.deliveryId).toBe('detached_completion:task-1');+ expect(second.deliveryId).toBe(first.deliveryId);+ expect(second.requestId).not.toBe(first.requestId);+ expect(first.event.payload).toEqual(+ expect.objectContaining({+ invocationId: 'trigger_original',+ generationCreatedAt: 1_786_999_999_000,+ wakeGenerationCreatedAt: 1_787_000_000_000,+ taskId: 'task-1',+ }),+ );+ });+});diff --git a/packages/api/src/agents/triggers/detachedAction.ts b/packages/api/src/agents/triggers/detachedAction.tsnew file mode 100644index 0000000000..1aee9e2ce5--- /dev/null+++ b/packages/api/src/agents/triggers/detachedAction.ts@@ -0,0 +1,473 @@+import { randomUUID } from 'node:crypto';+import {+ logger,+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+} from '@librechat/data-schemas';+import type {+ AgentEventActorDetachedAction,+ AgentTriggerDeliveryMethods,+ IAgentEventActorSuspensionEvidence,+} from '@librechat/data-schemas';+import type { EventActorInterrupt } from '@librechat/agents';+import type {+ AgentContinueTriggerEnvelope,+ AgentTriggerEnvelope,+ AgentTriggerExpectedAction,+} from './envelope';+import type { EventActorDetachedActionLifecycle } from '../handlers';+import type { AgentEventDetachedTerminalEvidence } from './types';+import type { SerializableJobData } from '~/stream';+import { matchesExpectedAction } from './expectedAction';+import { createAgentTriggerEnvelope } from './envelope';++const MAX_TERMINAL_RESULT_LENGTH = 32_768;+const TERMINAL_PERSIST_RETRY_INITIAL_MS = 100;+const TERMINAL_PERSIST_RETRY_MAX_MS = 30_000;+const RESERVATION_RECOVERY_MS = 60_000;+const RUNNING_RECOVERY_MS = 30 * 60_000;+const DETACHED_ACTION_PRODUCER_ENV = 'AGENT_TRIGGERS_DETACHED_ACTIONS_PRODUCER_ENABLED';+export const EVENT_ACTOR_DETACHED_COMPLETION_TYPE = 'librechat.event_actor.detached_completion';+export const EVENT_ACTOR_DETACHED_COMPLETION_SOURCE = 'librechat-event-actor';++export function isAgentEventActorDetachedActionProducerEnabled(): boolean {+ return process.env[DETACHED_ACTION_PRODUCER_ENV]?.trim().toLowerCase() === 'true';+}++export function parseAgentEventDetachedTerminalEvidence(+ value: unknown,+): AgentEventDetachedTerminalEvidence | undefined {+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {+ return undefined;+ }+ const input = value as Record<string, unknown>;+ const status = input.status;+ if (+ input.version !== 1 ||+ typeof input.deliveryKey !== 'string' ||+ input.deliveryKey.length === 0 ||+ input.deliveryKey.length > 128 ||+ !Number.isSafeInteger(input.generationCreatedAt) ||+ (input.generationCreatedAt as number) < 0 ||+ typeof input.taskId !== 'string' ||+ input.taskId.length === 0 ||+ input.taskId.length > 128 ||+ typeof input.idempotencyKey !== 'string' ||+ !/^[a-f0-9]{64}$/.test(input.idempotencyKey) ||+ (status !== 'succeeded' && status !== 'failed' && status !== 'cancelled') ||+ !Number.isSafeInteger(input.observedAt) ||+ (input.observedAt as number) < 0 ||+ (input.result != null &&+ (typeof input.result !== 'string' || input.result.length > MAX_TERMINAL_RESULT_LENGTH)) ||+ (input.error != null && (typeof input.error !== 'string' || input.error.length > 2_048)) ||+ (status === 'succeeded'+ ? typeof input.result !== 'string' || input.error != null+ : typeof input.error !== 'string' || input.result != null)+ ) {+ return undefined;+ }+ return {+ version: 1,+ deliveryKey: input.deliveryKey,+ generationCreatedAt: input.generationCreatedAt as number,+ taskId: input.taskId,+ idempotencyKey: input.idempotencyKey,+ status,+ ...(input.result == null ? {} : { result: input.result as string }),+ ...(input.error == null ? {} : { error: input.error as string }),+ observedAt: input.observedAt as number,+ };+}++export interface AgentEventActorDetachedCompletionProjection {+ version: 1;+ invocationId: string;+ /** Generation that owns the original delivery's detached-action record. */+ generationCreatedAt: number;+ /** Generation whose terminal handling enqueued this wake. */+ wakeGenerationCreatedAt: number;+ taskId: string;+ idempotencyKey: string;+}++export function parseAgentEventActorDetachedCompletion(+ value: unknown,+): AgentEventActorDetachedCompletionProjection | undefined {+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {+ return undefined;+ }+ const input = value as Record<string, unknown>;+ if (+ input.version !== 1 ||+ typeof input.invocationId !== 'string' ||+ input.invocationId.length === 0 ||+ input.invocationId.length > 128 ||+ !Number.isSafeInteger(input.generationCreatedAt) ||+ (input.generationCreatedAt as number) < 0 ||+ !Number.isSafeInteger(input.wakeGenerationCreatedAt) ||+ (input.wakeGenerationCreatedAt as number) < 0 ||+ typeof input.taskId !== 'string' ||+ input.taskId.length === 0 ||+ input.taskId.length > 128 ||+ typeof input.idempotencyKey !== 'string' ||+ !/^[a-f0-9]{64}$/.test(input.idempotencyKey)+ ) {+ return undefined;+ }+ return {+ version: 1,+ invocationId: input.invocationId,+ generationCreatedAt: input.generationCreatedAt as number,+ wakeGenerationCreatedAt: input.wakeGenerationCreatedAt as number,+ taskId: input.taskId,+ idempotencyKey: input.idempotencyKey,+ };+}++interface DetachedActionOwner {+ user: string;+ tenantId?: string;+ bindingId: string;+ conversationId: string;+ generationCreatedAt: number;+ turnCreatedAt: number;+ invocationId: string;+ expectedAction: AgentTriggerExpectedAction;+}++interface DetachedActionDependencies {+ reserveAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['reserveAgentEventActorDetachedAction'];+ markAgentEventActorDetachedActionRunning: AgentTriggerDeliveryMethods['markAgentEventActorDetachedActionRunning'];+ settleAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['settleAgentEventActorDetachedAction'];+ persistTerminalEvidence(input: AgentEventDetachedTerminalEvidence): Promise<void>;+ onTerminal(input: { taskId: string; idempotencyKey: string }): Promise<void>;+ waitForTerminalPersistenceRetry?(delayMs: number): Promise<void>;+ producerEnabled(): boolean;+ now?(): Date;+}++function waitForTerminalPersistenceRetry(delayMs: number): Promise<void> {+ return new Promise((resolve) => {+ const timer = setTimeout(resolve, delayMs);+ timer.unref();+ });+}++export interface AgentEventActorInternalSuspension {+ kind: 'internal_completion';+ actionId: string;+ jobCreatedAt: number;+ interrupt: EventActorInterrupt;+}++export interface AgentEventActorDetachedActionLifecycle extends EventActorDetachedActionLifecycle {+ readSuspension(): AgentEventActorInternalSuspension | undefined;+}++export interface AgentEventActorDetachedResumeInput {+ streamId: string;+ job: SerializableJobData;+ handlingGenerationCreatedAt: number;+ suspension: IAgentEventActorSuspensionEvidence;+ action: AgentEventActorDetachedAction;+}++interface DetachedResumeDependencies {+ getAgentTriggerDelivery: AgentTriggerDeliveryMethods['getAgentTriggerDelivery'];+ enqueueAgentTrigger(+ envelope: AgentTriggerEnvelope,+ options?: { requiredWorkerCapability?: string },+ ): Promise<unknown>;+ requestId?(): string;+ now?(): number;+}++function renderDetachedCompletion(action: AgentEventActorDetachedAction): string {+ const terminal =+ action.status === 'succeeded'+ ? `succeeded with this result:\n${action.result ?? ''}`+ : `${action.status} with this error:\n${action.error ?? 'No error detail was recorded.'}`;+ return [+ 'A detached expected action from your preceding event turn has now reached an authoritative terminal state.',+ `Tool: ${action.toolName}`,+ `Tool call: ${action.toolCallId}`,+ `The action ${terminal}`,+ 'Continue the same event invocation using this trusted internal completion. Do not launch the same action again.',+ ].join('\n');+}++/** Builds the exact internal continuation in the typed trigger layer. The app+ * server supplies only persistence and dispatch composition dependencies. */+export function createAgentEventDetachedResumeHandler(deps: DetachedResumeDependencies) {+ const nextRequestId = deps.requestId ?? randomUUID;+ const now = deps.now ?? Date.now;+ return async ({+ job,+ action,+ handlingGenerationCreatedAt,+ }: AgentEventActorDetachedResumeInput): Promise<void> => {+ const delivery = await deps.getAgentTriggerDelivery(job.agentEventDeliveryKey as string);+ const envelope = delivery?.envelope as Partial<AgentContinueTriggerEnvelope> | undefined;+ if (+ delivery == null ||+ String(delivery.user) !== job.userId ||+ (delivery.tenantId ?? undefined) !== job.tenantId ||+ envelope?.mode !== 'continue' ||+ envelope.principal?.userId !== job.userId ||+ (envelope.principal?.tenantId ?? undefined) !== job.tenantId ||+ envelope.target?.bindingId !== job.agentEventBindingId ||+ envelope.target?.conversationId !== job.conversationId ||+ envelope.expectedAction?.toolName !== action.expectedToolName+ ) {+ throw new Error('Detached Event Actor completion owner is unavailable');+ }+ const target = envelope.target as AgentContinueTriggerEnvelope['target'];+ const receivedAt = now();+ const continuation = createAgentTriggerEnvelope({+ mode: 'continue',+ requestId: nextRequestId(),+ deliveryId: `detached_completion:${action.taskId}`,+ receivedAt,+ principal: {+ id: job.userId,+ ...(job.tenantId == null ? {} : { tenantId: job.tenantId }),+ },+ event: {+ id: action.taskId,+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ occurredAt: action.settledAt?.getTime() ?? action.observedAt.getTime(),+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },+ payload: {+ version: 1,+ invocationId: job.agentEventDeliveryKey as string,+ generationCreatedAt: handlingGenerationCreatedAt,+ wakeGenerationCreatedAt: job.createdAt,+ taskId: action.taskId,+ idempotencyKey: action.idempotencyKey,+ },+ },+ target,+ input: renderDetachedCompletion(action),+ expectedAction: envelope.expectedAction,+ });+ await deps.enqueueAgentTrigger(continuation, {+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+ });+ };+}++function serializeTerminalResult(value: unknown): string {+ let serialized: string;+ if (typeof value === 'string') {+ serialized = value;+ } else {+ try {+ serialized = JSON.stringify(value) ?? String(value);+ } catch {+ serialized = String(value);+ }+ }+ return serialized.slice(0, MAX_TERMINAL_RESULT_LENGTH);+}++/**+ * Binds the generic background executor to one delivery-owned expected action.+ * The delivery row remains authoritative; request-local state only lets the+ * actor adapter discover the exact launch that this segment already persisted.+ */+export function createAgentEventActorDetachedActionLifecycle(+ owner: DetachedActionOwner,+ deps: DetachedActionDependencies,+): AgentEventActorDetachedActionLifecycle {+ const now = deps.now ?? (() => new Date());+ const producerEnabled = deps.producerEnabled();+ let current: { taskId: string; idempotencyKey: string; launchAcknowledged: boolean } | undefined;+ const scope = {+ deliveryKey: owner.invocationId,+ user: owner.user,+ ...(owner.tenantId == null ? {} : { tenantId: owner.tenantId }),+ bindingId: owner.bindingId,+ conversationId: owner.conversationId,+ generationCreatedAt: owner.generationCreatedAt,+ };+ const matchesCurrent = (input: { taskId: string; idempotencyKey: string }): boolean =>+ current?.taskId === input.taskId && current.idempotencyKey === input.idempotencyKey;++ return {+ async reserve(input) {+ if (+ !matchesExpectedAction(+ {+ toolName: input.toolName,+ toolCallId: input.toolCallId,+ arguments: input.arguments,+ },+ owner.expectedAction,+ )+ ) {+ return { status: 'ignored' };+ }+ if (!producerEnabled) {+ return {+ status: 'conflict',+ error:+ 'Detached Event Actor production is not activated for this deployment; no external action was launched',+ };+ }+ const reservedAt = now();+ const reservation = await deps.reserveAgentEventActorDetachedAction({+ ...scope,+ turnId: input.turnId,+ invocationId: owner.invocationId,+ expectedToolName: owner.expectedAction.toolName,+ toolName: input.toolName,+ toolCallId: input.toolCallId,+ reservedAt,+ recoveryAfter: new Date(reservedAt.getTime() + RESERVATION_RECOVERY_MS),+ });+ if (reservation.status === 'conflict') {+ return {+ status: 'conflict',+ error: 'A different detached action already owns this Event Actor invocation',+ };+ }+ if (+ reservation.status === 'replay' &&+ ['succeeded', 'failed', 'cancelled'].includes(reservation.action.status)+ ) {+ return {+ status: 'terminal',+ taskId: reservation.action.taskId,+ idempotencyKey: reservation.action.idempotencyKey,+ outcome: reservation.action.status as 'succeeded' | 'failed' | 'cancelled',+ ...(reservation.action.result == null ? {} : { result: reservation.action.result }),+ ...(reservation.action.error == null ? {} : { error: reservation.action.error }),+ };+ }+ if (reservation.action.status === 'launch_indeterminate') {+ return {+ status: 'conflict',+ error:+ 'The detached Event Actor launch outcome is indeterminate; exact terminal proof is required before this invocation can continue',+ };+ }+ if (reservation.status === 'replay') {+ if (+ matchesCurrent({+ taskId: reservation.action.taskId,+ idempotencyKey: reservation.action.idempotencyKey,+ })+ ) {+ return {+ status: 'replay',+ taskId: reservation.action.taskId,+ idempotencyKey: reservation.action.idempotencyKey,+ };+ }+ return {+ status: 'conflict',+ error:+ reservation.action.status === 'running'+ ? 'The detached Event Actor action was already launched by another executor'+ : 'The detached Event Actor launch acknowledgement is still pending',+ };+ }+ current = {+ taskId: reservation.action.taskId,+ idempotencyKey: reservation.action.idempotencyKey,+ launchAcknowledged: false,+ };+ return {+ status: reservation.status,+ taskId: current.taskId,+ idempotencyKey: current.idempotencyKey,+ };+ },+ async markRunning(input) {+ if (!matchesCurrent(input)) {+ return false;+ }+ const observedAt = now();+ const marked = await deps.markAgentEventActorDetachedActionRunning({+ ...scope,+ ...input,+ observedAt,+ recoveryAfter: new Date(observedAt.getTime() + RUNNING_RECOVERY_MS),+ });+ const accepted = marked.status !== 'conflict';+ if (accepted && current != null) {+ current.launchAcknowledged = true;+ }+ return accepted;+ },+ async settle(input) {+ if (!matchesCurrent(input)) {+ return false;+ }+ const observedAt = now();+ const evidence: AgentEventDetachedTerminalEvidence = {+ version: 1,+ deliveryKey: owner.invocationId,+ generationCreatedAt: owner.generationCreatedAt,+ taskId: input.taskId,+ idempotencyKey: input.idempotencyKey,+ status: input.status,+ ...(input.status === 'succeeded'+ ? { result: serializeTerminalResult(input.result) }+ : { error: String(input.error ?? 'Detached action failed').slice(0, 2_048) }),+ observedAt: observedAt.getTime(),+ };+ const waitForRetry = deps.waitForTerminalPersistenceRetry ?? waitForTerminalPersistenceRetry;+ let retryDelayMs = TERMINAL_PERSIST_RETRY_INITIAL_MS;+ let staged = false;+ for (;;) {+ try {+ if (!staged) {+ await deps.persistTerminalEvidence(evidence);+ staged = true;+ }+ const settled = await deps.settleAgentEventActorDetachedAction({+ ...scope,+ taskId: evidence.taskId,+ idempotencyKey: evidence.idempotencyKey,+ status: evidence.status,+ ...(evidence.result == null ? {} : { result: evidence.result }),+ ...(evidence.error == null ? {} : { error: evidence.error }),+ observedAt,+ });+ return settled.status !== 'conflict';+ } catch (error) {+ logger.warn('[event-actor] Retrying detached terminal evidence persistence', {+ taskId: input.taskId,+ retryDelayMs,+ error: error instanceof Error ? error.message : String(error),+ });+ await waitForRetry(retryDelayMs);+ retryDelayMs = Math.min(retryDelayMs * 2, TERMINAL_PERSIST_RETRY_MAX_MS);+ }+ }+ },+ async wake(input) {+ if (matchesCurrent(input)) {+ await deps.onTerminal(input);+ }+ },+ readSuspension() {+ if (current?.launchAcknowledged !== true) {+ return undefined;+ }+ return {+ kind: 'internal_completion',+ actionId: current.taskId,+ jobCreatedAt: owner.turnCreatedAt,+ interrupt: {+ id: current.taskId,+ payload: {+ type: 'event_actor_detached_action',+ taskId: current.taskId,+ idempotencyKey: current.idempotencyKey,+ },+ },+ };+ },+ };+}diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.tsindex 502702a6c9..d6c3ad19ca 100644--- a/packages/api/src/agents/triggers/engine.ts+++ b/packages/api/src/agents/triggers/engine.ts@@ -61,10 +61,14 @@ export class AgentTriggerDeliveryDeferredError extends Error {export type AgentTriggerDeliveryStatus =| 'staging'+ | 'capability_staging'| 'batched'| 'pending'+ | 'capability_pending'| 'leased'+ | 'capability_leased'| 'succeeded'+ | 'capability_dead'| 'dead';export interface AgentTriggerDeliveryFailure {diff --git a/packages/api/src/agents/triggers/expectedAction.ts b/packages/api/src/agents/triggers/expectedAction.tsnew file mode 100644index 0000000000..0e1b66953a--- /dev/null+++ b/packages/api/src/agents/triggers/expectedAction.ts@@ -0,0 +1,54 @@+import type { AgentTriggerExpectedAction } from './envelope';++export interface CompletedToolEvidence {+ toolName: string;+ toolCallId?: string;+ arguments?: unknown;+}++export function parseAgentExpectedActionArguments(value: unknown): unknown {+ if (typeof value !== 'string') {+ return value;+ }+ try {+ return JSON.parse(value) as unknown;+ } catch {+ return value;+ }+}++function containsSubset(value: unknown, subset: unknown): boolean {+ if (Array.isArray(subset)) {+ return (+ Array.isArray(value) &&+ value.length === subset.length &&+ subset.every((expected, index) => containsSubset(value[index], expected))+ );+ }+ if (subset == null || typeof subset !== 'object') {+ return Object.is(value, subset);+ }+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {+ return false;+ }+ return Object.entries(subset).every(([key, expected]) =>+ containsSubset((value as Record<string, unknown>)[key], expected),+ );+}++export function matchesExpectedAction(+ evidence: CompletedToolEvidence,+ expected: AgentTriggerExpectedAction,+): boolean {+ const nameMatches =+ evidence.toolName === expected.toolName ||+ evidence.toolName.startsWith(`${expected.toolName}_mcp_`);+ return (+ nameMatches &&+ (expected.argumentSubset == null ||+ containsSubset(+ parseAgentExpectedActionArguments(evidence.arguments),+ expected.argumentSubset,+ ))+ );+}diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.tsindex 7f10674a41..706fa0056c 100644--- a/packages/api/src/agents/triggers/host.spec.ts+++ b/packages/api/src/agents/triggers/host.spec.ts@@ -1,6 +1,10 @@import { Constants, EModelEndpoint } from 'librechat-data-provider';import { getRequestId, getTenantId, getUserId } from '@librechat/data-schemas';import type { AgentTriggerExecutionHostDeps, AgentTriggerFetch } from './host';+import {+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+} from './detachedAction';import { createAgentTriggerEnvelope, getAgentTriggerIdempotencyKey } from './envelope';import { AgentTriggerExecutionError, createAgentTriggerExecutionHost } from './host';@@ -625,6 +629,57 @@ describe('createAgentTriggerExecutionHost continue adapter', () => {).not.toHaveProperty('payload');});+ it('projects only the bounded internal detached-completion authority', async () => {+ const base = createContinueEnvelope();+ if (base.mode !== 'continue') {+ throw new Error('Expected a continue envelope');+ }+ const completion = {+ version: 1 as const,+ invocationId: 'original-delivery-1',+ generationCreatedAt: 1_787_000_000_000,+ wakeGenerationCreatedAt: 1_787_000_000_000,+ taskId: 'event-actor-task-1',+ idempotencyKey: 'a'.repeat(64),+ };+ const envelope = {+ ...base,+ target: {+ ...base.target,+ bindingId: 'binding-1',+ sourceKeyId: 'source-key-1',+ },+ event: {+ id: completion.taskId,+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ occurredAt: completion.generationCreatedAt + 1,+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },+ payload: completion,+ },+ expectedAction: { toolName: 'submit_move' },+ };+ const fetcher = fetchMock(async () =>+ response({+ streamId: 'conversation-1',+ conversationId: 'conversation-1',+ generationCreatedAt: completion.generationCreatedAt + 2,+ status: 'started',+ }),+ );+ const getBaseUrl = jest.fn(() => 'http://127.0.0.1:3080');++ await createAgentTriggerExecutionHost(deps(fetcher, { getBaseUrl })).dispatch(envelope);++ expect(getBaseUrl).toHaveBeenCalledWith({ localOnly: true });+ const body = JSON.parse(String(fetcher.mock.calls[0][1]?.body));+ expect(body.clientRequestId).toBe(getAgentTriggerIdempotencyKey(envelope));+ expect(body.agentEventDelivery).toMatchObject({+ deliveryKey: getAgentTriggerIdempotencyKey(envelope),+ internalCompletion: completion,+ });+ expect(body.agentEventDelivery.event).not.toHaveProperty('payload');+ });+it.each(['PARENT_NOT_READY', 'EVENT_ACTOR_NOT_READY'])('retries without consuming the logical delivery for temporary admission code %s',async (code) => {diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.tsindex a1477ab1b0..341041f27e 100644--- a/packages/api/src/agents/triggers/host.ts+++ b/packages/api/src/agents/triggers/host.ts@@ -9,6 +9,11 @@ import type {} from './envelope';import type { AgentTriggerDispatchContext } from './dispatch';import type { AgentRunPrincipal } from '../envelope';+import {+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ parseAgentEventActorDetachedCompletion,+} from './detachedAction';import { dispatchAgentTrigger } from './dispatch';const DEFAULT_FIRE_TIMEOUT_MS = 30_000;@@ -119,7 +124,7 @@ export type AgentTriggerExecutionResult =export interface AgentTriggerExecutionHostDeps {/** Trusted root URL for this LibreChat server. */- getBaseUrl: () => string;+ getBaseUrl: (options?: { localOnly?: boolean }) => string;/** Mint a short-lived token for the envelope's already-authenticated principal. */mintToken: (principal: AgentRunPrincipal, envelope: AgentTriggerEnvelope) => MaybePromise<string>;/** Optional user-timezone resolver for dynamic date variables in a new run. */@@ -534,6 +539,13 @@ async function startRun(timeoutMs: number,): Promise<AgentTriggerContinueResult | AgentTriggerFireResult> {const mode = envelope.mode;+ const detachedCompletion =+ mode === 'continue' &&+ envelope.event.type === EVENT_ACTOR_DETACHED_COMPLETION_TYPE &&+ envelope.event.source.type === 'internal' &&+ envelope.event.source.id === EVENT_ACTOR_DETACHED_COMPLETION_SOURCE+ ? parseAgentEventActorDetachedCompletion(envelope.event.payload)+ : undefined;const scope = abortScope(context.signal, timeoutMs);let preparation: AgentTriggerContinuePreparation | undefined;try {@@ -574,7 +586,12 @@ async function startRun(scope,context.signal,),- setupValue(() => deps.getBaseUrl(), mode, scope, context.signal),+ setupValue(+ () => deps.getBaseUrl(detachedCompletion == null ? undefined : { localOnly: true }),+ mode,+ scope,+ context.signal,+ ),]).catch((error: unknown) => {scope.abort();throw error;@@ -632,6 +649,7 @@ async function startRun(...(envelope.expectedAction != null && {expectedAction: envelope.expectedAction,}),+ ...(detachedCompletion == null ? {} : { internalCompletion: detachedCompletion }),},}),...(envelope.mode === 'fire' && {diff --git a/packages/api/src/agents/triggers/index.ts b/packages/api/src/agents/triggers/index.tsindex 16b47f8d85..742e8ac79c 100644--- a/packages/api/src/agents/triggers/index.ts+++ b/packages/api/src/agents/triggers/index.ts@@ -3,6 +3,7 @@ export * from './bindings';export * from './actor';export * from './bindingResolver';export * from './delivery';+export * from './detachedAction';export * from './envelope';export * from './host';export * from './ingress';diff --git a/packages/api/src/agents/triggers/outcome.spec.ts b/packages/api/src/agents/triggers/outcome.spec.tsindex e30e4e6890..89a2647b70 100644--- a/packages/api/src/agents/triggers/outcome.spec.ts+++ b/packages/api/src/agents/triggers/outcome.spec.ts@@ -19,22 +19,32 @@ const createAgentEventTerminalHandler = ('settleAgentTriggerHandlingOutcome'> &Partial<Parameters<typeof createAgentEventTerminalHandlerImpl>[0]>,+ options?: Parameters<typeof createAgentEventTerminalHandlerImpl>[1],) =>- createAgentEventTerminalHandlerImpl({- getAgentEventActorSnapshot: jest.fn().mockResolvedValue(undefined),- getMessage: jest.fn().mockResolvedValue(null),- resolveAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),- clearAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),- settleAgentEventActorReceipt: jest.fn().mockResolvedValue(true),- getAgentEventActorReceipt: jest.fn().mockResolvedValue(null),- backfillAgentEventActorReceipt: jest.fn().mockResolvedValue(true),- completeAgentEventActorLegacyTurn: jest.fn().mockResolvedValue(true),- cancelAgentEventActorSuspension: jest.fn().mockResolvedValue({ status: 'cancelled' }),- releaseAgentEventActorAction: jest.fn().mockResolvedValue(true),- getAgentEventActorActionAdmission: jest.fn().mockResolvedValue(null),- hasAgentEventActorActionAdmission: jest.fn().mockResolvedValue(false),- ...methods,- });+ createAgentEventTerminalHandlerImpl(+ {+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue(undefined),+ recordAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),+ getMessage: jest.fn().mockResolvedValue(null),+ resolveAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),+ clearAgentEventActorReconciliation: jest.fn().mockResolvedValue(true),+ settleAgentEventActorReceipt: jest.fn().mockResolvedValue(true),+ getAgentEventActorReceipt: jest.fn().mockResolvedValue(null),+ backfillAgentEventActorReceipt: jest.fn().mockResolvedValue(true),+ completeAgentEventActorLegacyTurn: jest.fn().mockResolvedValue(true),+ cancelAgentEventActorSuspension: jest.fn().mockResolvedValue({ status: 'cancelled' }),+ releaseAgentEventActorAction: jest.fn().mockResolvedValue(true),+ getAgentEventActorActionAdmission: jest.fn().mockResolvedValue(null),+ hasAgentEventActorActionAdmission: jest.fn().mockResolvedValue(false),+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(null),+ settleAgentEventActorDetachedAction: jest.fn().mockResolvedValue({ status: 'applied' }),+ markAgentEventActorDetachedActionLaunchIndeterminate: jest+ .fn()+ .mockResolvedValue({ status: 'applied' }),+ ...methods,+ },+ options,+ );function job(overrides: Partial<SerializableJobData> = {}): SerializableJobData {return {@@ -107,6 +117,994 @@ describe('agent event terminal outcomes', () => {mockedCancelAgentEventActor.mockResolvedValue({ status: 'cancelled' });});+ it('hands a terminal internal suspension to the durable resume adapter without settling', async () => {+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);+ const resumeDetachedAction = jest.fn().mockResolvedValue(undefined);+ const suspension = {+ version: 1 as const,+ suspensionId: 'suspension-detached-1',+ attempt: 0,+ issuedAt: 1,+ expiresAt: 2,+ invocation: {+ actorThreadId: 'conversation-1',+ invocationId: 'trigger_1',+ depth: 0,+ continuation: 'warm' as const,+ base: { actorThreadId: 'conversation-1', generation: 1 },+ fork: {+ threadId: 'conversation-1',+ checkpointNs: 'event-actor/trigger-1',+ checkpointId: 'checkpoint-1',+ invocationId: 'trigger_1',+ },+ },+ checkpoint: {+ threadId: 'conversation-1',+ checkpointNs: 'event-actor/trigger-1',+ checkpointId: 'checkpoint-1',+ invocationId: 'trigger_1',+ },+ interrupt: { id: 'task-1', payload: { type: 'event_actor_detached_action' } },+ suspensionDigest: 'signed-digest',+ };+ const detachedAction = {+ version: 1 as const,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-1',+ taskId: 'task-1',+ idempotencyKey: 'a'.repeat(64),+ launchAttempt: 0 as const,+ status: 'succeeded' as const,+ reservedAt: new Date(),+ recoveryAfter: new Date(),+ launchedAt: new Date(),+ settledAt: new Date(),+ observedAt: new Date(),+ result: 'move accepted',+ };+ const settleAgentEventActorDetachedAction = jest+ .fn()+ .mockResolvedValue({ status: 'already_achieved' });+ const handler = createAgentEventTerminalHandler(+ {+ settleAgentTriggerHandlingOutcome,+ settleAgentEventActorDetachedAction,+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension,+ actionId: 'task-1',+ jobCreatedAt: 1_787_000_000_000,+ status: 'pending',+ observedAt: new Date(),+ },+ }),+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(detachedAction),+ },+ { resumeDetachedAction },+ );++ await handler(+ 'conversation-1',+ job({+ agentEventBindingId: 'binding-1',+ agentEventDetachedTerminalEvidence: {+ version: 1,+ deliveryKey: 'trigger_1',+ generationCreatedAt: 1_787_000_000_000,+ taskId: detachedAction.taskId,+ idempotencyKey: detachedAction.idempotencyKey,+ status: 'succeeded',+ result: 'move accepted',+ observedAt: 1_787_000_000_500,+ },+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: 0,+ },+ }),+ [],+ );++ expect(resumeDetachedAction).toHaveBeenCalledWith(+ expect.objectContaining({ streamId: 'conversation-1', suspension, action: detachedAction }),+ );+ expect(settleAgentEventActorDetachedAction).toHaveBeenCalledWith(+ expect.objectContaining({+ deliveryKey: 'trigger_1',+ generationCreatedAt: 1_787_000_000_000,+ taskId: detachedAction.taskId,+ status: 'succeeded',+ result: 'move accepted',+ observedAt: new Date(1_787_000_000_500),+ }),+ );+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();+ });++ it('durably enqueues a re-pause successor before settling its completion predecessor', async () => {+ const events: string[] = [];+ const suspension = suspensionEvidence('suspension-repause-successor', 1);+ const action = {+ version: 1 as const,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-successor',+ taskId: 'task-successor',+ idempotencyKey: '9'.repeat(64),+ launchAttempt: 1,+ status: 'succeeded' as const,+ reservedAt: new Date(),+ recoveryAfter: new Date(),+ observedAt: new Date(),+ settledAt: new Date(),+ result: 'accepted',+ };+ const settleAgentTriggerHandlingOutcome = jest.fn(async () => {+ events.push('settle-predecessor');+ return true;+ });+ const resumeDetachedAction = jest.fn(async () => {+ events.push('enqueue-successor');+ });+ const handler = createAgentEventTerminalHandler(+ {+ settleAgentTriggerHandlingOutcome,+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(action),+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension,+ actionId: action.taskId,+ jobCreatedAt: 1_787_000_010_000,+ handlingGenerationCreatedAt: 1_787_000_000_000,+ status: 'pending',+ observedAt: new Date(),+ },+ }),+ },+ { resumeDetachedAction },+ );++ await handler(+ 'conversation-1',+ job({+ createdAt: 1_787_000_010_000,+ agentEventDeliveryKey: 'trigger_completion_1',+ agentEventInvocationKey: 'trigger_1',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: suspension.attempt,+ },+ }),+ [],+ );++ expect(events).toEqual(['enqueue-successor', 'settle-predecessor']);+ expect(resumeDetachedAction).toHaveBeenCalledWith(+ expect.objectContaining({ handlingGenerationCreatedAt: 1_787_000_000_000 }),+ );+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ deliveryKey: 'trigger_completion_1',+ generationCreatedAt: 1_787_000_010_000,+ status: 'completed_no_action',+ }),+ );+ });++ it('preserves original ownership and retires every predecessor across three completion hops', async () => {+ const events: string[] = [];+ const originalGenerationCreatedAt = 1_787_000_000_000;+ const suspensions = [+ suspensionEvidence('suspension-hop-1', 1),+ suspensionEvidence('suspension-hop-2', 2),+ ];+ const actions = suspensions.map((_, index) => ({+ version: 1 as const,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: `call-hop-${index + 1}`,+ taskId: `task-hop-${index + 1}`,+ idempotencyKey: String(index + 1).repeat(64),+ launchAttempt: index + 1,+ status: 'succeeded' as const,+ reservedAt: new Date(),+ recoveryAfter: new Date(),+ observedAt: new Date(),+ settledAt: new Date(),+ result: 'accepted',+ }));+ const getAgentEventActorSnapshot = jest+ .fn()+ .mockResolvedValueOnce({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension: suspensions[0],+ actionId: actions[0].taskId,+ jobCreatedAt: originalGenerationCreatedAt + 10_000,+ handlingGenerationCreatedAt: originalGenerationCreatedAt,+ status: 'pending',+ observedAt: new Date(),+ },+ })+ .mockResolvedValueOnce({+ state: null,+ epoch: 2,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension: suspensions[1],+ actionId: actions[1].taskId,+ jobCreatedAt: originalGenerationCreatedAt + 20_000,+ handlingGenerationCreatedAt: originalGenerationCreatedAt,+ status: 'pending',+ observedAt: new Date(),+ },+ });+ const getAgentEventActorDetachedAction = jest+ .fn()+ .mockResolvedValueOnce(actions[0])+ .mockResolvedValueOnce(actions[1]);+ const settleAgentTriggerHandlingOutcome = jest.fn(async (input) => {+ events.push(`settle:${input.deliveryKey}`);+ return true;+ });+ const resumeDetachedAction = jest.fn(async (input) => {+ events.push(`enqueue:${input.action.taskId}`);+ });+ const handler = createAgentEventTerminalHandler(+ {+ settleAgentTriggerHandlingOutcome,+ getAgentEventActorSnapshot,+ getAgentEventActorDetachedAction,+ },+ { resumeDetachedAction },+ );++ for (let index = 0; index < 2; index++) {+ await handler(+ 'conversation-1',+ job({+ createdAt: originalGenerationCreatedAt + (index + 1) * 10_000,+ agentEventDeliveryKey: `trigger_completion_${index + 1}`,+ agentEventInvocationKey: 'trigger_1',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: {+ version: 1,+ suspensionId: suspensions[index].suspensionId,+ attempt: suspensions[index].attempt,+ },+ }),+ [],+ );+ }++ expect(events).toEqual([+ 'enqueue:task-hop-1',+ 'settle:trigger_completion_1',+ 'enqueue:task-hop-2',+ 'settle:trigger_completion_2',+ ]);+ expect(getAgentEventActorDetachedAction).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({ generationCreatedAt: originalGenerationCreatedAt }),+ );+ expect(getAgentEventActorDetachedAction).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ generationCreatedAt: originalGenerationCreatedAt }),+ );+ expect(resumeDetachedAction).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({ handlingGenerationCreatedAt: originalGenerationCreatedAt }),+ );+ });++ it('repairs a crash after successor enqueue by replaying enqueue and predecessor settlement', async () => {+ const suspension = suspensionEvidence('suspension-handoff-retry', 1);+ const action = {+ version: 1 as const,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-handoff-retry',+ taskId: 'task-handoff-retry',+ idempotencyKey: '6'.repeat(64),+ launchAttempt: 1,+ status: 'succeeded' as const,+ reservedAt: new Date(),+ recoveryAfter: new Date(),+ observedAt: new Date(),+ settledAt: new Date(),+ result: 'accepted',+ };+ const snapshot = {+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion' as const,+ suspension,+ actionId: action.taskId,+ jobCreatedAt: 1_787_000_010_000,+ handlingGenerationCreatedAt: 1_787_000_000_000,+ status: 'pending' as const,+ observedAt: new Date(),+ },+ };+ const resumeDetachedAction = jest.fn().mockResolvedValue(undefined);+ const settleAgentTriggerHandlingOutcome = jest+ .fn()+ .mockResolvedValueOnce(false)+ .mockResolvedValueOnce(true);+ const handler = createAgentEventTerminalHandler(+ {+ settleAgentTriggerHandlingOutcome,+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue(action),+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue(snapshot),+ },+ { resumeDetachedAction },+ );+ const completionJob = job({+ createdAt: 1_787_000_010_000,+ agentEventDeliveryKey: 'trigger_completion_retry',+ agentEventInvocationKey: 'trigger_1',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: suspension.attempt,+ },+ });++ await expect(handler('conversation-1', completionJob, [])).rejects.toThrow(+ 'Failed to settle internal completion delivery',+ );+ await expect(handler('conversation-1', completionJob, [])).resolves.toBeUndefined();+ expect(resumeDetachedAction).toHaveBeenCalledTimes(2);+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledTimes(2);+ });++ it('settles both the original invocation and its internal completion delivery', async () => {+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);+ const getAgentEventActorDetachedAction = jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-failed',+ taskId: 'task-failed',+ idempotencyKey: 'b'.repeat(64),+ launchAttempt: 0,+ status: 'failed',+ reservedAt: new Date(),+ recoveryAfter: new Date(),+ observedAt: new Date(),+ settledAt: new Date(),+ error: 'move service rejected the request',+ });+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ getAgentEventActorDetachedAction,+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension: {+ version: 1,+ suspensionId: 'suspension-failed',+ attempt: 0,+ invocation: { invocationId: 'trigger_1' },+ },+ actionId: 'task-failed',+ jobCreatedAt: 1_787_000_000_000,+ handlingGenerationCreatedAt: 1_787_000_000_000,+ status: 'closed',+ outcome: 'settled',+ observedAt: new Date(),+ },+ }),+ });++ await handler(+ 'conversation-1',+ job({+ createdAt: 1_787_000_010_000,+ agentEventDeliveryKey: 'trigger_completion_1',+ agentEventInvocationKey: 'trigger_1',+ agentEventBindingId: 'binding-1',+ }),+ [],+ );++ expect(getAgentEventActorDetachedAction).toHaveBeenCalledWith(+ expect.objectContaining({ generationCreatedAt: 1_787_000_000_000 }),+ );+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenNthCalledWith(+ 1,+ expect.objectContaining({+ deliveryKey: 'trigger_1',+ generationCreatedAt: 1_787_000_000_000,+ status: 'failed',+ error: 'move service rejected the request',+ }),+ );+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenNthCalledWith(+ 2,+ expect.objectContaining({+ deliveryKey: 'trigger_completion_1',+ generationCreatedAt: 1_787_000_010_000,+ status: 'failed',+ error: 'move service rejected the request',+ }),+ );+ });++ it('marks an expired detached launch indeterminate without resuming or relaunching', async () => {+ const suspension = suspensionEvidence('suspension-indeterminate');+ const getAgentEventActorDetachedAction = jest+ .fn()+ .mockResolvedValueOnce({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-indeterminate',+ taskId: 'task-indeterminate',+ idempotencyKey: 'c'.repeat(64),+ launchAttempt: 0,+ status: 'running',+ reservedAt: new Date(0),+ observedAt: new Date(0),+ recoveryAfter: new Date(1),+ })+ .mockResolvedValueOnce({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-indeterminate',+ taskId: 'task-indeterminate',+ idempotencyKey: 'c'.repeat(64),+ launchAttempt: 0,+ status: 'launch_indeterminate',+ reservedAt: new Date(0),+ observedAt: new Date(),+ recoveryAfter: new Date(1),+ });+ const markIndeterminate = jest.fn().mockResolvedValue({ status: 'applied' });+ const resumeDetachedAction = jest.fn();+ const handler = createAgentEventTerminalHandler(+ {+ settleAgentTriggerHandlingOutcome: jest.fn(),+ getAgentEventActorDetachedAction,+ markAgentEventActorDetachedActionLaunchIndeterminate: markIndeterminate,+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension,+ actionId: 'task-indeterminate',+ jobCreatedAt: 1_787_000_000_000,+ status: 'pending',+ observedAt: new Date(),+ },+ }),+ },+ { resumeDetachedAction },+ );++ await expect(+ handler(+ 'conversation-1',+ job({+ agentEventBindingId: 'binding-1',+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: suspension.attempt,+ },+ }),+ [],+ ),+ ).rejects.toThrow('detached action launch is indeterminate');+ expect(markIndeterminate).toHaveBeenCalledTimes(1);+ expect(resumeDetachedAction).not.toHaveBeenCalled();+ });++ it('keeps an indeterminate action owner open when the executor dies before suspension', async () => {+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);+ const markIndeterminate = jest.fn().mockResolvedValue({ status: 'applied' });+ const action = {+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-unacknowledged',+ taskId: 'task-unacknowledged',+ idempotencyKey: 'e'.repeat(64),+ launchAttempt: 0,+ status: 'running',+ reservedAt: new Date(0),+ observedAt: new Date(0),+ recoveryAfter: new Date(1),+ };+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ markAgentEventActorDetachedActionLaunchIndeterminate: markIndeterminate,+ getAgentEventActorDetachedAction: jest+ .fn()+ .mockResolvedValueOnce(action)+ .mockResolvedValueOnce({ ...action, status: 'launch_indeterminate' }),+ });++ await expect(+ handler(+ 'conversation-1',+ job({ status: 'error', error: 'executor exited', agentEventBindingId: 'binding-1' }),+ [],+ ),+ ).rejects.toThrow('detached action launch is indeterminate');++ expect(markIndeterminate).toHaveBeenCalledTimes(1);+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();+ });++ it('keeps an unexpired detached launch open after generation failure', async () => {+ const settleAgentTriggerHandlingOutcome = jest.fn();+ const markIndeterminate = jest.fn();+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ markAgentEventActorDetachedActionLaunchIndeterminate: markIndeterminate,+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-running',+ taskId: 'task-running',+ idempotencyKey: '8'.repeat(64),+ launchAttempt: 0,+ status: 'running',+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(Date.now() + 60_000),+ }),+ });++ await expect(+ handler(+ 'conversation-1',+ job({ status: 'error', error: 'generation failed', agentEventBindingId: 'binding-1' }),+ [],+ ),+ ).rejects.toThrow('detached action is still in flight');+ expect(markIndeterminate).not.toHaveBeenCalled();+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();+ });++ it('cancels an aborted internal completion instead of waking it', async () => {+ const suspension = suspensionEvidence('suspension-aborted-internal');+ const resumeDetachedAction = jest.fn();+ const handler = createAgentEventTerminalHandler(+ {+ settleAgentTriggerHandlingOutcome: jest.fn().mockResolvedValue(true),+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-aborted',+ taskId: 'task-aborted',+ idempotencyKey: 'd'.repeat(64),+ launchAttempt: 0,+ status: 'failed',+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ settledAt: new Date(),+ error: 'detached action failed',+ }),+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension,+ actionId: 'task-aborted',+ jobCreatedAt: 1_787_000_000_000,+ status: 'pending',+ observedAt: new Date(),+ },+ }),+ },+ { resumeDetachedAction },+ );++ await handler(+ 'conversation-1',+ job({+ status: 'aborted',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: suspension.attempt,+ },+ }),+ [],+ );++ expect(resumeDetachedAction).not.toHaveBeenCalled();+ expect(mockedCancelAgentEventActor).toHaveBeenCalledTimes(1);+ });++ it('retires a failed detached action that became terminal before suspension storage', async () => {+ const checkpoint = {+ threadId: 'conversation-1',+ checkpointNs: 'event-actor/trigger_1',+ };+ const initialSnapshot = {+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [+ {+ invocationId: 'trigger_1',+ actionAdmitted: true,+ status: 'invocation_pending' as const,+ checkpoint,+ action: { toolName: 'submit_move' },+ observedAt: new Date(),+ },+ ],+ };+ const retiredSnapshot = { ...initialSnapshot, reconciliations: [] };+ const getAgentEventActorSnapshot = jest+ .fn()+ .mockResolvedValueOnce(initialSnapshot)+ .mockResolvedValue(retiredSnapshot);+ const resolveAgentEventActorReconciliation = jest.fn().mockResolvedValue(true);+ const getAgentEventActorActionAdmission = jest+ .fn()+ .mockResolvedValueOnce('admission-pre-suspension')+ .mockResolvedValue(null);+ const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true);+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ getAgentEventActorSnapshot,+ resolveAgentEventActorReconciliation,+ getAgentEventActorActionAdmission,+ releaseAgentEventActorAction,+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-pre-suspension-failure',+ taskId: 'task-pre-suspension-failure',+ idempotencyKey: 'f'.repeat(64),+ launchAttempt: 0,+ status: 'failed',+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ settledAt: new Date(),+ error: 'detached action failed before suspension storage',+ }),+ });+ const terminalJob = job({ agentEventBindingId: 'binding-1' });++ await handler('conversation-1', terminalJob, []);+ await handler('conversation-1', terminalJob, []);++ expect(resolveAgentEventActorReconciliation).toHaveBeenCalledTimes(1);+ expect(resolveAgentEventActorReconciliation).toHaveBeenCalledWith({+ user: 'user-1',+ conversationId: 'conversation-1',+ invocationId: 'trigger_1',+ checkpoint,+ expectedActionAdmitted: true,+ resolution: 'invocation_abandoned',+ });+ expect(releaseAgentEventActorAction).toHaveBeenCalledTimes(1);+ expect(releaseAgentEventActorAction).toHaveBeenCalledWith(+ expect.objectContaining({+ deliveryKey: 'trigger_1',+ bindingId: 'binding-1',+ admissionId: 'admission-pre-suspension',+ }),+ );+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledTimes(2);+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenLastCalledWith(+ expect.objectContaining({+ deliveryKey: 'trigger_1',+ status: 'failed',+ error: 'detached action failed before suspension storage',+ }),+ );+ });++ it('retires the claimed predecessor when a successor fails before re-pause storage', async () => {+ const predecessor = suspensionEvidence('suspension-claimed-predecessor', 1);+ const initialSnapshot = {+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [+ {+ invocationId: 'trigger_1',+ actionAdmitted: true,+ status: 'invocation_pending' as const,+ checkpoint: predecessor.checkpoint,+ action: { toolName: 'submit_move' },+ observedAt: new Date(),+ },+ ],+ suspension: {+ kind: 'internal_completion' as const,+ suspension: predecessor,+ actionId: 'task-predecessor',+ jobCreatedAt: 1_787_000_000_000,+ handlingGenerationCreatedAt: 1_787_000_000_000,+ status: 'claimed' as const,+ resumeAttemptId: 'trigger_completion_failed_successor',+ observedAt: new Date(),+ },+ };+ const closedSnapshot = {+ ...initialSnapshot,+ reconciliations: [],+ suspension: {+ ...initialSnapshot.suspension,+ status: 'closed' as const,+ outcome: 'cancelled' as const,+ },+ };+ const getAgentEventActorSnapshot = jest+ .fn()+ .mockResolvedValueOnce(initialSnapshot)+ .mockResolvedValue(closedSnapshot);+ const releaseAgentEventActorAction = jest+ .fn()+ .mockRejectedValueOnce(new Error('delivery store unavailable'))+ .mockResolvedValue(true);+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ getAgentEventActorSnapshot,+ releaseAgentEventActorAction,+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-failed-successor',+ taskId: 'task-failed-successor',+ idempotencyKey: '6'.repeat(64),+ launchAttempt: 1,+ status: 'failed',+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ settledAt: new Date(),+ error: 'successor failed before re-pause storage',+ }),+ });++ const terminalJob = job({+ createdAt: 1_787_000_010_000,+ agentEventDeliveryKey: 'trigger_completion_failed_successor',+ agentEventInvocationKey: 'trigger_1',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: undefined,+ });++ await expect(handler('conversation-1', terminalJob, [])).rejects.toThrow(+ 'delivery store unavailable',+ );+ await handler('conversation-1', terminalJob, []);++ expect(mockedCancelAgentEventActor).toHaveBeenCalledWith(+ expect.objectContaining({+ suspension: predecessor,+ claimedResumeAttemptId: 'trigger_completion_failed_successor',+ reason: 'cancelled',+ }),+ expect.any(Object),+ );+ expect(releaseAgentEventActorAction).toHaveBeenCalledTimes(2);+ expect(releaseAgentEventActorAction).toHaveBeenLastCalledWith(+ expect.objectContaining({ deliveryKey: 'trigger_1', bindingId: 'binding-1' }),+ );+ expect(settleAgentTriggerHandlingOutcome).toHaveBeenCalledTimes(2);+ expect(+ settleAgentTriggerHandlingOutcome.mock.calls.map(([input]) => input.deliveryKey),+ ).toEqual(['trigger_1', 'trigger_completion_failed_successor']);+ expect(settleAgentTriggerHandlingOutcome.mock.calls[0][0]).toMatchObject({+ status: 'failed',+ error: 'successor failed before re-pause storage',+ });+ });++ it('releases admission after an exact completion predecessor settled normally', async () => {+ const predecessor = suspensionEvidence('suspension-settled-predecessor', 1);+ const completionDeliveryKey = 'trigger_completion_settled_successor';+ const releaseAgentEventActorAction = jest.fn().mockResolvedValue(true);+ const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ releaseAgentEventActorAction,+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [],+ suspension: {+ kind: 'internal_completion',+ suspension: predecessor,+ actionId: 'task-settled-predecessor',+ jobCreatedAt: 1_787_000_000_000,+ handlingGenerationCreatedAt: 1_787_000_000_000,+ status: 'closed',+ outcome: 'settled',+ resumeAttemptId: completionDeliveryKey,+ observedAt: new Date(),+ },+ }),+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-settled-successor',+ turnId: 'response-settled-successor:0',+ taskId: 'task-settled-successor',+ idempotencyKey: '8'.repeat(64),+ launchAttempt: 1,+ status: 'succeeded',+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ settledAt: new Date(),+ result: 'accepted',+ }),+ });++ await handler(+ 'conversation-1',+ job({+ createdAt: 1_787_000_010_000,+ agentEventDeliveryKey: completionDeliveryKey,+ agentEventInvocationKey: 'trigger_1',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: undefined,+ }),+ [],+ );++ expect(releaseAgentEventActorAction).toHaveBeenCalledWith(+ expect.objectContaining({+ deliveryKey: 'trigger_1',+ bindingId: 'binding-1',+ admissionId: expect.any(String),+ }),+ );+ expect(+ settleAgentTriggerHandlingOutcome.mock.calls.map(([input]) => input.deliveryKey),+ ).toEqual(['trigger_1', completionDeliveryKey]);+ });++ it('records reconciliation when a detached action succeeds after generation abort', async () => {+ const suspension = suspensionEvidence('suspension-aborted-success');+ const recordAgentEventActorReconciliation = jest.fn().mockResolvedValue(true);+ const settleAgentTriggerHandlingOutcome = jest.fn();+ const handler = createAgentEventTerminalHandler({+ settleAgentTriggerHandlingOutcome,+ recordAgentEventActorReconciliation,+ getAgentEventActorDetachedAction: jest.fn().mockResolvedValue({+ version: 1,+ invocationId: 'trigger_1',+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-aborted-success',+ taskId: 'task-aborted-success',+ idempotencyKey: '7'.repeat(64),+ launchAttempt: 0,+ status: 'succeeded',+ reservedAt: new Date(),+ observedAt: new Date(),+ recoveryAfter: new Date(),+ settledAt: new Date(),+ result: 'accepted',+ }),+ getAgentEventActorSnapshot: jest.fn().mockResolvedValue({+ state: null,+ epoch: 1,+ legacyTurn: null,+ reconciliations: [+ {+ invocationId: 'trigger_1',+ actionAdmitted: true,+ status: 'invocation_pending',+ checkpoint: suspension.checkpoint,+ action: { toolName: 'submit_move' },+ observedAt: new Date(),+ },+ ],+ suspension: {+ kind: 'internal_completion',+ suspension,+ actionId: 'task-aborted-success',+ jobCreatedAt: job().createdAt,+ status: 'pending',+ observedAt: new Date(),+ },+ }),+ });++ await expect(+ handler(+ 'conversation-1',+ job({+ status: 'aborted',+ agentEventBindingId: 'binding-1',+ agentEventSuspension: {+ version: 1,+ suspensionId: suspension.suspensionId,+ attempt: suspension.attempt,+ },+ }),+ [],+ ),+ ).rejects.toThrow('requires commit_indeterminate reconciliation');+ expect(recordAgentEventActorReconciliation).toHaveBeenCalledWith(+ expect.objectContaining({+ reconciliation: expect.objectContaining({+ invocationId: 'trigger_1',+ status: 'commit_indeterminate',+ action: {+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-aborted-success',+ },+ }),+ }),+ );+ expect(mockedCancelAgentEventActor).not.toHaveBeenCalled();+ expect(settleAgentTriggerHandlingOutcome).not.toHaveBeenCalled();+ });+it('cancels a versioned paused actor before settling its expired delivery', async () => {const settleAgentTriggerHandlingOutcome = jest.fn().mockResolvedValue(true);const suspension = {diff --git a/packages/api/src/agents/triggers/outcome.ts b/packages/api/src/agents/triggers/outcome.tsindex 199625765c..387d93bcf8 100644--- a/packages/api/src/agents/triggers/outcome.ts+++ b/packages/api/src/agents/triggers/outcome.ts@@ -1,16 +1,23 @@import type {+ AgentEventActorDetachedAction,AgentTriggerDeliveryMethods,ConversationMethods,IAgentEventActorSuspension,MessageMethods,} from '@librechat/data-schemas';import type { Agents } from 'librechat-data-provider';+import type { AgentEventActorDetachedResumeInput } from './detachedAction';+import type { CompletedToolEvidence } from './expectedAction';import type { AgentTriggerExpectedAction } from './envelope';import type { AgentEventAppliedAction } from './types';import type { SerializableJobData } from '~/stream';+import { matchesExpectedAction, parseAgentExpectedActionArguments } from './expectedAction';import { cancelAgentEventActor, createAgentEventActorActionAdmissionId } from './actor';+import { parseAgentEventDetachedTerminalEvidence } from './detachedAction';export type { AgentEventAppliedAction } from './types';+export { matchesExpectedAction } from './expectedAction';+export type { CompletedToolEvidence } from './expectedAction';interface SettleAgentTriggerHandlingOutcomeInput {deliveryKey: string;@@ -22,12 +29,6 @@ interface SettleAgentTriggerHandlingOutcomeInput {action?: { toolName: string; toolCallId?: string };}-interface CompletedToolEvidence {- toolName: string;- toolCallId?: string;- arguments?: unknown;-}-export interface AgentEventRunOutcome {status: 'applied' | 'completed_no_action' | 'failed' | 'cancelled';action?: { toolName: string; toolCallId?: string };@@ -55,7 +56,7 @@ async function hasDurableAgentEventHistory(input: {}function isBackgroundNonExecutionReceipt(value: unknown, argumentsValue: unknown): boolean {- const parsedArguments = parseArguments(argumentsValue);+ const parsedArguments = parseAgentExpectedActionArguments(argumentsValue);if (parsedArguments == null ||typeof parsedArguments !== 'object' ||@@ -64,7 +65,7 @@ function isBackgroundNonExecutionReceipt(value: unknown, argumentsValue: unknown) {return false;}- const parsed = parseArguments(value);+ const parsed = parseAgentExpectedActionArguments(value);if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {return false;}@@ -152,17 +153,6 @@ function nonExecutedHITLToolCallIds(return ids;}-function parseArguments(value: unknown): unknown {- if (typeof value !== 'string') {- return value;- }- try {- return JSON.parse(value) as unknown;- } catch {- return value;- }-}-/** Classifies terminal run evidence once for both checkpoint commit and public receipt. */export function classifyAgentEventRunOutcome(job: SerializableJobData,@@ -206,39 +196,6 @@ export function findAgentEventAppliedAction(};}-function containsSubset(value: unknown, subset: unknown): boolean {- if (Array.isArray(subset)) {- return (- Array.isArray(value) &&- value.length === subset.length &&- subset.every((expected, index) => containsSubset(value[index], expected))- );- }- if (subset == null || typeof subset !== 'object') {- return Object.is(value, subset);- }- if (value == null || typeof value !== 'object' || Array.isArray(value)) {- return false;- }- return Object.entries(subset).every(([key, expected]) =>- containsSubset((value as Record<string, unknown>)[key], expected),- );-}--function matchesExpectedAction(- evidence: CompletedToolEvidence,- expected: AgentTriggerExpectedAction,-): boolean {- const nameMatches =- evidence.toolName === expected.toolName ||- evidence.toolName.startsWith(`${expected.toolName}_mcp_`);- return (- nameMatches &&- (expected.argumentSubset == null ||- containsSubset(parseArguments(evidence.arguments), expected.argumentSubset))- );-}-export interface AgentEventActionRecorder {observeToolEnd(data: {input?: unknown;@@ -281,7 +238,7 @@ export function createAgentEventActionRecorder(* would be indistinguishable from a real result, so a call the model* detached can never qualify through this shape. */if (data.outputFiltered === true) {- const parsedInput = parseArguments(data.input);+ const parsedInput = parseAgentExpectedActionArguments(data.input);if (parsedInput != null &&typeof parsedInput === 'object' &&@@ -332,23 +289,32 @@ export function createAgentEventActionRecorder(};}-export function createAgentEventTerminalHandler(methods: {- settleAgentTriggerHandlingOutcome: (- input: SettleAgentTriggerHandlingOutcomeInput,- ) => Promise<boolean>;- getAgentEventActorSnapshot: ConversationMethods['getAgentEventActorSnapshot'];- resolveAgentEventActorReconciliation: ConversationMethods['resolveAgentEventActorReconciliation'];- clearAgentEventActorReconciliation: ConversationMethods['clearAgentEventActorReconciliation'];- settleAgentEventActorReceipt: AgentTriggerDeliveryMethods['settleAgentEventActorReceipt'];- getAgentEventActorReceipt: AgentTriggerDeliveryMethods['getAgentEventActorReceipt'];- backfillAgentEventActorReceipt: AgentTriggerDeliveryMethods['backfillAgentEventActorReceipt'];- completeAgentEventActorLegacyTurn: ConversationMethods['completeAgentEventActorLegacyTurn'];- cancelAgentEventActorSuspension: ConversationMethods['cancelAgentEventActorSuspension'];- releaseAgentEventActorAction: AgentTriggerDeliveryMethods['releaseAgentEventActorAction'];- getAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['getAgentEventActorActionAdmission'];- hasAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['hasAgentEventActorActionAdmission'];- getMessage: MessageMethods['getMessage'];-}): (+export function createAgentEventTerminalHandler(+ methods: {+ settleAgentTriggerHandlingOutcome: (+ input: SettleAgentTriggerHandlingOutcomeInput,+ ) => Promise<boolean>;+ getAgentEventActorSnapshot: ConversationMethods['getAgentEventActorSnapshot'];+ recordAgentEventActorReconciliation: ConversationMethods['recordAgentEventActorReconciliation'];+ resolveAgentEventActorReconciliation: ConversationMethods['resolveAgentEventActorReconciliation'];+ clearAgentEventActorReconciliation: ConversationMethods['clearAgentEventActorReconciliation'];+ settleAgentEventActorReceipt: AgentTriggerDeliveryMethods['settleAgentEventActorReceipt'];+ getAgentEventActorReceipt: AgentTriggerDeliveryMethods['getAgentEventActorReceipt'];+ backfillAgentEventActorReceipt: AgentTriggerDeliveryMethods['backfillAgentEventActorReceipt'];+ completeAgentEventActorLegacyTurn: ConversationMethods['completeAgentEventActorLegacyTurn'];+ cancelAgentEventActorSuspension: ConversationMethods['cancelAgentEventActorSuspension'];+ releaseAgentEventActorAction: AgentTriggerDeliveryMethods['releaseAgentEventActorAction'];+ getAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['getAgentEventActorActionAdmission'];+ hasAgentEventActorActionAdmission: AgentTriggerDeliveryMethods['hasAgentEventActorActionAdmission'];+ getAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['getAgentEventActorDetachedAction'];+ settleAgentEventActorDetachedAction: AgentTriggerDeliveryMethods['settleAgentEventActorDetachedAction'];+ markAgentEventActorDetachedActionLaunchIndeterminate: AgentTriggerDeliveryMethods['markAgentEventActorDetachedActionLaunchIndeterminate'];+ getMessage: MessageMethods['getMessage'];+ },+ options: {+ resumeDetachedAction?(input: AgentEventActorDetachedResumeInput): Promise<void>;+ } = {},+): (streamId: string,job: SerializableJobData,runSteps: Agents.RunStep[],@@ -356,17 +322,50 @@ export function createAgentEventTerminalHandler(methods: {) => Promise<void> {return async (streamId: string,- job: SerializableJobData,+ receivedJob: SerializableJobData,runSteps: Agents.RunStep[],content: Agents.MessageContentComplex[] = [],) => {- if (job.agentEventDeliveryKey == null) {+ if (receivedJob.agentEventDeliveryKey == null) {return;}+ const completionDeliveryKey =+ receivedJob.agentEventInvocationKey == null ? undefined : receivedJob.agentEventDeliveryKey;+ /** Actor state remains owned by the original invocation. The internal+ * completion delivery separately owns this generation's mailbox lane. */+ const job: SerializableJobData & { agentEventDeliveryKey: string } =+ receivedJob.agentEventInvocationKey == null+ ? { ...receivedJob, agentEventDeliveryKey: receivedJob.agentEventDeliveryKey }+ : {+ ...receivedJob,+ agentEventDeliveryKey: receivedJob.agentEventInvocationKey,+ };const conversationId = job.conversationId ?? streamId;const outcome = classifyAgentEventRunOutcome(job, runSteps, content);const settledAt = new Date(job.completedAt ?? Date.now());+ const settleCompletionDelivery = async (+ completionOutcome: AgentEventRunOutcome,+ failureError = receivedJob.error ?? 'Generation failed',+ ): Promise<void> => {+ if (completionDeliveryKey == null) {+ return;+ }+ const completionSettled = await methods.settleAgentTriggerHandlingOutcome({+ deliveryKey: completionDeliveryKey,+ conversationId,+ generationCreatedAt: receivedJob.createdAt,+ status: completionOutcome.status,+ settledAt,+ ...(completionOutcome.status === 'failed' && { error: failureError }),+ ...(completionOutcome.action != null && { action: completionOutcome.action }),+ });+ if (!completionSettled) {+ throw new Error(`Failed to settle internal completion delivery ${completionDeliveryKey}`);+ }+ };let committedAction: AgentEventAppliedAction | undefined;+ let detachedTerminalFailure: string | undefined;+ let detachedTerminalRetiredWithoutSuspension = false;let compensated = false;let actorReceiptSettled = false;const owner = {@@ -378,6 +377,255 @@ export function createAgentEventTerminalHandler(methods: {let retiredWithoutAction: IAgentEventActorSuspension | undefined;const isIrrecoverablyTerminal = job.status === 'aborted' || job.status === 'error';const unprojectedSuspension = snapshot?.suspension;+ const handlingGenerationCreatedAt =+ unprojectedSuspension?.suspension.invocation.invocationId === job.agentEventDeliveryKey+ ? (unprojectedSuspension.handlingGenerationCreatedAt ?? job.createdAt)+ : job.createdAt;+ let detachedSuspensionAction: AgentEventActorDetachedAction | null = null;+ if (job.agentEventBindingId != null) {+ const detachedActionOwner = {+ deliveryKey: job.agentEventDeliveryKey,+ user: job.userId,+ ...(job.tenantId == null ? {} : { tenantId: job.tenantId }),+ bindingId: job.agentEventBindingId,+ conversationId,+ generationCreatedAt: handlingGenerationCreatedAt,+ };+ const retainedTerminalEvidence = parseAgentEventDetachedTerminalEvidence(+ job.agentEventDetachedTerminalEvidence,+ );+ if (job.agentEventDetachedTerminalEvidence != null && retainedTerminalEvidence == null) {+ throw new Error('Detached Event Actor terminal retry evidence is invalid');+ }+ if (retainedTerminalEvidence != null) {+ if (+ retainedTerminalEvidence.deliveryKey !== job.agentEventDeliveryKey ||+ retainedTerminalEvidence.generationCreatedAt !== handlingGenerationCreatedAt+ ) {+ throw new Error('Detached Event Actor terminal retry evidence is stale');+ }+ const replayed = await methods.settleAgentEventActorDetachedAction({+ ...detachedActionOwner,+ taskId: retainedTerminalEvidence.taskId,+ idempotencyKey: retainedTerminalEvidence.idempotencyKey,+ status: retainedTerminalEvidence.status,+ ...(retainedTerminalEvidence.result == null+ ? {}+ : { result: retainedTerminalEvidence.result }),+ ...(retainedTerminalEvidence.error == null+ ? {}+ : { error: retainedTerminalEvidence.error }),+ observedAt: new Date(retainedTerminalEvidence.observedAt),+ });+ if (replayed.status === 'conflict') {+ throw new Error('Detached Event Actor terminal retry evidence conflicts with action');+ }+ }+ detachedSuspensionAction =+ await methods.getAgentEventActorDetachedAction(detachedActionOwner);+ const recoveryObservedAt = new Date();+ if (+ detachedSuspensionAction != null &&+ ['reserved', 'running'].includes(detachedSuspensionAction.status) &&+ detachedSuspensionAction.recoveryAfter <= recoveryObservedAt+ ) {+ await methods.markAgentEventActorDetachedActionLaunchIndeterminate({+ ...detachedActionOwner,+ taskId: detachedSuspensionAction.taskId,+ idempotencyKey: detachedSuspensionAction.idempotencyKey,+ observedAt: recoveryObservedAt,+ });+ /** Re-read after the CAS so a concurrent exact terminal callback wins+ * over recovery and can immediately continue the suspended actor. */+ detachedSuspensionAction =+ await methods.getAgentEventActorDetachedAction(detachedActionOwner);+ }+ if (+ detachedSuspensionAction?.status === 'failed' ||+ detachedSuspensionAction?.status === 'cancelled' ||+ detachedSuspensionAction?.status === 'launch_indeterminate'+ ) {+ detachedTerminalFailure =+ detachedSuspensionAction.error ??+ (detachedSuspensionAction.status === 'launch_indeterminate'+ ? 'Detached expected action launch is indeterminate'+ : `Detached expected action ${detachedSuspensionAction.status}`);+ }+ if (+ job.agentEventSuspension == null &&+ completionDeliveryKey != null &&+ (detachedSuspensionAction?.status === 'failed' ||+ detachedSuspensionAction?.status === 'cancelled') &&+ unprojectedSuspension?.status === 'claimed' &&+ unprojectedSuspension.resumeAttemptId === completionDeliveryKey &&+ unprojectedSuspension.suspension.invocation.invocationId === job.agentEventDeliveryKey+ ) {+ /** A resumed hop can persist exact negative terminal evidence before+ * replacing its claimed predecessor with the successor suspension.+ * The completion delivery key is also the resume-attempt fence, so it+ * authorizes retiring only this generation's claimed predecessor. */+ retiredWithoutAction = unprojectedSuspension;+ const cancellation = await cancelAgentEventActor(+ {+ ...owner,+ suspension: unprojectedSuspension.suspension,+ cancelAttemptId: `terminal:${job.createdAt}`,+ reason: 'cancelled',+ claimedResumeAttemptId: completionDeliveryKey,+ },+ { cancelSuspension: methods.cancelAgentEventActorSuspension },+ );+ if (cancellation.status !== 'cancelled') {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached predecessor cancellation is indeterminate`,+ );+ }+ snapshot = await methods.getAgentEventActorSnapshot(owner);+ }+ /** An aborted/error generation does not prove its detached side effect+ * stopped. Retain the original delivery and suspension until exact+ * terminal evidence arrives or the recovery fence records uncertainty. */+ if (+ isIrrecoverablyTerminal &&+ detachedSuspensionAction != null &&+ ['reserved', 'running'].includes(detachedSuspensionAction.status)+ ) {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached action is still in flight`,+ );+ }+ if (detachedSuspensionAction?.status === 'launch_indeterminate') {+ /** Quarantine is action truth, not a generation failure. Keep the+ * original delivery open so a late exact callback remains admissible;+ * no generic retry or ordinary cancellation may erase the uncertainty. */+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached action launch is indeterminate`,+ );+ }+ if (isIrrecoverablyTerminal && detachedSuspensionAction?.status === 'succeeded') {+ /** The side effect is authoritative but this aborted generation owns+ * no committed actor checkpoint. Persist the existing reconciliation+ * contract and wait for explicit compensation/repair instead of+ * falsely reporting either cancellation or an applied actor turn. */+ const marker = snapshot?.reconciliations.find(+ (item) => item.invocationId === job.agentEventDeliveryKey,+ );+ if (marker?.status === 'settled' && marker.resolution === 'action_compensated') {+ detachedTerminalFailure = 'Detached action was explicitly compensated';+ } else {+ if (marker?.status === 'invocation_pending') {+ const recorded = await methods.recordAgentEventActorReconciliation({+ ...owner,+ reconciliation: {+ invocationId: job.agentEventDeliveryKey,+ actionAdmitted: marker.actionAdmitted,+ status: 'commit_indeterminate',+ checkpoint: marker.checkpoint,+ action: {+ toolName: detachedSuspensionAction.toolName,+ toolCallId: detachedSuspensionAction.toolCallId,+ },+ error: 'Detached action succeeded after its generation terminated',+ observedAt:+ detachedSuspensionAction.settledAt ?? detachedSuspensionAction.observedAt,+ },+ });+ if (!recorded) {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached reconciliation could not be recorded`,+ );+ }+ } else if (marker?.status !== 'commit_indeterminate') {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached reconciliation owner is unavailable`,+ );+ }+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached success requires commit_indeterminate reconciliation`,+ );+ }+ }+ if (+ unprojectedSuspension == null &&+ (detachedSuspensionAction?.status === 'failed' ||+ detachedSuspensionAction?.status === 'cancelled')+ ) {+ /** Exact negative terminal evidence proves the detached side effect no+ * longer owns a future actor resume. Retire the pre-suspension+ * invocation fence before releasing delivery-side admission; replay+ * can repair either half independently. */+ const marker = snapshot?.reconciliations.find(+ (item) => item.invocationId === job.agentEventDeliveryKey,+ );+ if (marker?.status === 'invocation_pending') {+ const abandoned = await methods.resolveAgentEventActorReconciliation({+ ...owner,+ invocationId: job.agentEventDeliveryKey,+ checkpoint: marker.checkpoint,+ expectedActionAdmitted: true,+ resolution: 'invocation_abandoned',+ });+ snapshot = await methods.getAgentEventActorSnapshot(owner);+ const remaining = snapshot?.reconciliations.find(+ (item) => item.invocationId === job.agentEventDeliveryKey,+ );+ if (!abandoned && remaining?.status === 'invocation_pending') {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached terminal lifecycle could not be retired`,+ );+ }+ detachedTerminalRetiredWithoutSuspension = remaining == null;+ } else if (marker == null) {+ /** Re-entry after the conversation-side retirement committed but+ * before delivery admission was released. */+ detachedTerminalRetiredWithoutSuspension = true;+ }+ }+ }+ if (+ unprojectedSuspension?.kind === 'internal_completion' &&+ unprojectedSuspension.status === 'pending' &&+ unprojectedSuspension.suspension.invocation.invocationId === job.agentEventDeliveryKey &&+ !isIrrecoverablyTerminal+ ) {+ const projection = job.agentEventSuspension;+ if (+ projection == null ||+ projection.suspensionId !== unprojectedSuspension.suspension.suspensionId ||+ projection.attempt !== unprojectedSuspension.suspension.attempt ||+ job.agentEventBindingId == null+ ) {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} internal suspension projection is stale`,+ );+ }+ const action = detachedSuspensionAction;+ if (action?.status === 'launch_indeterminate') {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached action launch is indeterminate`,+ );+ }+ if (action == null || !['succeeded', 'failed', 'cancelled'].includes(action.status)) {+ throw new Error(+ `Agent event actor ${job.agentEventDeliveryKey} detached action is not terminal`,+ );+ }+ if (options.resumeDetachedAction == null) {+ throw new Error('Detached Event Actor resume adapter is unavailable');+ }+ await options.resumeDetachedAction({+ streamId,+ job,+ handlingGenerationCreatedAt,+ suspension: unprojectedSuspension.suspension,+ action,+ });+ /** The successor is durable before its predecessor lane is retired. A+ * crash between these writes leaves the successor safely blocked until+ * idempotent terminal replay settles this completion delivery. */+ await settleCompletionDelivery(outcome);+ return;+ }if (job.agentEventSuspension == null &&isIrrecoverablyTerminal &&@@ -490,17 +738,28 @@ export function createAgentEventTerminalHandler(methods: {* admission was not yet released. Only exact closed no-action evidence is* eligible; a committed suspension represents an applied action. */const closed = snapshot?.suspension;+ const closedDetachedPredecessor =+ job.agentEventSuspension == null &&+ completionDeliveryKey != null &&+ closed?.status === 'closed' &&+ (closed.outcome === 'settled' ||+ (closed.outcome === 'cancelled' &&+ (detachedSuspensionAction?.status === 'failed' ||+ detachedSuspensionAction?.status === 'cancelled'))) &&+ closed.resumeAttemptId === completionDeliveryKey &&+ closed.suspension.invocation.invocationId === job.agentEventDeliveryKey;if (retiredWithoutAction == null &&closed?.status === 'closed' &&(closed.outcome === 'settled' || closed.outcome === 'cancelled') &&- closed.jobCreatedAt === job.createdAt &&closed.suspension.invocation.invocationId === job.agentEventDeliveryKey &&- (closed.outcome === 'settled'- ? closed.resumeAttemptId != null && closed.resumeAttemptId === job.providerExecutionId- : isIrrecoverablyTerminal &&- (closed.resumeAttemptId == null ||- closed.resumeAttemptId !== job.providerExecutionStartedId))+ (closedDetachedPredecessor ||+ (closed.jobCreatedAt === job.createdAt &&+ (closed.outcome === 'settled'+ ? closed.resumeAttemptId != null && closed.resumeAttemptId === job.providerExecutionId+ : isIrrecoverablyTerminal &&+ (closed.resumeAttemptId == null ||+ closed.resumeAttemptId !== job.providerExecutionStartedId))))) {retiredWithoutAction = closed;}@@ -513,8 +772,7 @@ export function createAgentEventTerminalHandler(methods: {);if (retiredAdmissionId == null &&- snapshot == null &&- isIrrecoverablyTerminal &&+ ((snapshot == null && isIrrecoverablyTerminal) || detachedTerminalRetiredWithoutSuspension) &&job.agentEventBindingId != null) {retiredAdmissionId = await methods.getAgentEventActorActionAdmission({@@ -570,7 +828,7 @@ export function createAgentEventTerminalHandler(methods: {...(job.tenantId == null ? {} : { tenantId: job.tenantId }),bindingId: durableReceipt.bindingId,conversationId,- generationCreatedAt: job.createdAt,+ generationCreatedAt: handlingGenerationCreatedAt,status: durableReceipt.resolution === 'action_compensated' ? 'failed' : 'applied',settledAt: durableReceipt.settledAt,...(durableReceipt.resolution === 'action_compensated' && {@@ -626,7 +884,7 @@ export function createAgentEventTerminalHandler(methods: {const terminalized = await methods.settleAgentTriggerHandlingOutcome({deliveryKey: job.agentEventDeliveryKey,conversationId,- generationCreatedAt: job.createdAt,+ generationCreatedAt: handlingGenerationCreatedAt,status: legacyStatus,settledAt: lifecycle.observedAt,...(legacyError == null ? {} : { error: legacyError }),@@ -643,7 +901,7 @@ export function createAgentEventTerminalHandler(methods: {...(job.tenantId == null ? {} : { tenantId: job.tenantId }),bindingId: job.agentEventBindingId,conversationId,- generationCreatedAt: job.createdAt,+ generationCreatedAt: handlingGenerationCreatedAt,status: legacyStatus,settledAt: lifecycle.observedAt,...(legacyError == null ? {} : { error: legacyError }),@@ -704,7 +962,7 @@ export function createAgentEventTerminalHandler(methods: {...(job.tenantId == null ? {} : { tenantId: job.tenantId }),bindingId: job.agentEventBindingId,conversationId,- generationCreatedAt: job.createdAt,+ generationCreatedAt: handlingGenerationCreatedAt,status: 'applied',settledAt,...(lifecycle.actionAdmitted === true && { requiresActionAdmission: true }),@@ -803,6 +1061,8 @@ export function createAgentEventTerminalHandler(methods: {settlementOutcome = { status: 'failed' };} else if (committedAction != null) {settlementOutcome = { status: 'applied', action: committedAction };+ } else if (detachedTerminalFailure != null) {+ settlementOutcome = { status: 'failed' };}if (job.agentEventLegacyTurnToken != null && snapshot?.legacyTurn != null) {if (snapshot.legacyTurn.token !== job.agentEventLegacyTurnToken) {@@ -832,23 +1092,35 @@ export function createAgentEventTerminalHandler(methods: {}}if (actorReceiptSettled) {+ await settleCompletionDelivery(+ settlementOutcome,+ compensated+ ? 'Applied event actor action was explicitly compensated'+ : (detachedTerminalFailure ?? receivedJob.error ?? 'Generation failed'),+ );return;}const settled = await methods.settleAgentTriggerHandlingOutcome({deliveryKey: job.agentEventDeliveryKey,conversationId,- generationCreatedAt: job.createdAt,+ generationCreatedAt: handlingGenerationCreatedAt,status: settlementOutcome.status,settledAt,...(settlementOutcome.status === 'failed' && {error: compensated? 'Applied event actor action was explicitly compensated'- : (job.error ?? 'Generation failed'),+ : (detachedTerminalFailure ?? job.error ?? 'Generation failed'),}),...(settlementOutcome.action != null && { action: settlementOutcome.action }),});if (!settled) {throw new Error(`Failed to settle agent event delivery ${job.agentEventDeliveryKey}`);}+ await settleCompletionDelivery(+ settlementOutcome,+ compensated+ ? 'Applied event actor action was explicitly compensated'+ : (detachedTerminalFailure ?? receivedJob.error ?? 'Generation failed'),+ );};}diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsindex 22e5bf57a2..de458161e3 100644--- a/packages/api/src/agents/triggers/service.delivery.spec.ts+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -1,4 +1,8 @@-import { getTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas';+import {+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+ getTenantId,+ SYSTEM_TENANT_ID,+} from '@librechat/data-schemas';import type { AgentTriggerDeliveryPersistence, AgentTriggerStoredRecord } from './service';import { AgentTriggerServiceUnavailableError, createAgentTriggerService } from './service';import { __resetShutdownStateForTests } from '../../app/shutdown';@@ -157,6 +161,7 @@ describe('durable agent trigger service', () => {methods,mintToken: () => 'token',fetch: async () => new Response('{}', { status: 500 }),+ supportsDetachedActionCompletion: () => true,deliveryOptions: { concurrency: 1, tickMs: 60_000 },});@@ -168,6 +173,11 @@ describe('durable agent trigger service', () => {expect(methods.ensureAgentTriggerDeliveryIndexes).toHaveBeenCalledTimes(1);expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(+ expect.objectContaining({+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],+ }),+ );expect(methods.enqueueAgentTriggerDelivery).toHaveBeenCalledWith(expect.objectContaining({deliveryKey: expect.stringMatching(/^trigger_/),@@ -187,6 +197,22 @@ describe('durable agent trigger service', () => {await service.stop();});+ it('does not advertise detached completion capability without durable generation storage', async () => {+ const methods = deliveryMethods();+ const service = createAgentTriggerService({+ methods,+ supportsDetachedActionCompletion: () => false,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });++ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });++ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(+ expect.objectContaining({ workerCapabilities: [] }),+ );+ await service.stop();+ });+it('persists terminal handling serialization only for bound continuations', async () => {const methods = deliveryMethods();const service = createAgentTriggerService({diff --git a/packages/api/src/agents/triggers/service.spec.ts b/packages/api/src/agents/triggers/service.spec.tsindex 51fe993731..846004b557 100644--- a/packages/api/src/agents/triggers/service.spec.ts+++ b/packages/api/src/agents/triggers/service.spec.ts@@ -1,5 +1,9 @@import jwt from 'jsonwebtoken';import type { AgentTriggerFetch } from './host';+import {+ EVENT_ACTOR_DETACHED_COMPLETION_SOURCE,+ EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+} from './detachedAction';import { createAgentTriggerEnvelope } from './envelope';import { AGENT_TRIGGER_SCOPE } from '../../crypto/jwt';import { createAgentTriggerService } from './service';@@ -22,6 +26,38 @@ const envelope = () =>input: 'Handle the ready resource.',});+const detachedCompletionEnvelope = () =>+ createAgentTriggerEnvelope({+ mode: 'continue',+ requestId: 'request-detached-1',+ deliveryId: 'delivery-detached-1',+ receivedAt: 20,+ principal: { id: 'user-1', role: 'member', tenantId: 'tenant-1' },+ target: {+ agentId: 'agent-1',+ conversationId: 'conversation-1',+ parentMessageId: 'parent-1',+ bindingId: 'binding-1',+ sourceKeyId: 'source-key-1',+ },+ event: {+ id: 'event_actor_task_1',+ type: EVENT_ACTOR_DETACHED_COMPLETION_TYPE,+ occurredAt: 10,+ source: { id: EVENT_ACTOR_DETACHED_COMPLETION_SOURCE, type: 'internal' },+ payload: {+ version: 1,+ invocationId: 'trigger-original-1',+ generationCreatedAt: 1,+ wakeGenerationCreatedAt: 2,+ taskId: 'event_actor_task_1',+ idempotencyKey: 'a'.repeat(64),+ },+ },+ input: 'Resume the actor.',+ expectedAction: { toolName: 'submit_move' },+ });+const accepted = () =>new Response(JSON.stringify({@@ -91,6 +127,19 @@ describe('createAgentTriggerService', () => {);});+ it('keeps detached completions on the capable local replica despite a self-URL override', async () => {+ process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal/base';+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(+ async () => accepted(),+ );+ const service = createAgentTriggerService({ fetch: fetcher, mintToken: () => 'token' });+ await service.initialize({ address: { address: '0.0.0.0', family: 'IPv4', port: 4123 } });++ await service.dispatch(detachedCompletionEnvelope());++ expect(String(fetcher.mock.calls[0][0])).toBe('http://127.0.0.1:4123/api/agents/chat/agents');+ });+it('rechecks the durable principal before dispatching queued or direct work', async () => {process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal';const fetcher = jest.fn(async () => accepted());diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsindex 549630c0fe..554471c291 100644--- a/packages/api/src/agents/triggers/service.ts+++ b/packages/api/src/agents/triggers/service.ts@@ -1,4 +1,8 @@-import { logger, runAsSystem } from '@librechat/data-schemas';+import {+ AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+ logger,+ runAsSystem,+} from '@librechat/data-schemas';import type { AgentTriggerDeliveryStatusRecord } from '@librechat/data-schemas';import type {AgentTriggerDeliveryFailure,@@ -45,6 +49,7 @@ export interface AgentTriggerServiceDeps {userDrainPollMs?: number;purgeRecoveryIntervalMs?: number;purgeRecoveryLimit?: number;+ supportsDetachedActionCompletion?: () => boolean;}export interface AgentTriggerDeliveryReceipt {@@ -86,7 +91,11 @@ export interface AgentTriggerDeliveryPersistence {enqueueAgentTriggerDelivery: (input: PreparedAgentTriggerDelivery,) => Promise<{ delivery: AgentTriggerStoredRecord; replayed: boolean }>;- claimNextAgentTriggerDelivery: AgentTriggerDeliveryStore['claimNext'];+ claimNextAgentTriggerDelivery: (+ input: Parameters<AgentTriggerDeliveryStore['claimNext']>[0] & {+ workerCapabilities?: string[];+ },+ ) => ReturnType<AgentTriggerDeliveryStore['claimNext']>;findEarlierAgentTriggerDelivery: AgentTriggerDeliveryStore['findEarlierUnsettled'];getAgentTriggerDeliveryBatch: AgentTriggerDeliveryStore['getBatch'];releaseAgentTriggerDelivery: AgentTriggerDeliveryStore['release'];@@ -148,9 +157,18 @@ export interface AgentTriggerService {purgeUser: (userId: string) => Promise<void>;}-function createDeliveryStore(methods: AgentTriggerDeliveryPersistence): AgentTriggerDeliveryStore {+function createDeliveryStore(+ methods: AgentTriggerDeliveryPersistence,+ supportsDetachedActionCompletion: () => boolean,+): AgentTriggerDeliveryStore {return {- claimNext: methods.claimNextAgentTriggerDelivery,+ claimNext: (input) =>+ methods.claimNextAgentTriggerDelivery({+ ...input,+ workerCapabilities: supportsDetachedActionCompletion()+ ? [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1]+ : [],+ }),findEarlierUnsettled: methods.findEarlierAgentTriggerDelivery,getBatch: methods.getAgentTriggerDeliveryBatch,release: methods.releaseAgentTriggerDelivery,@@ -165,7 +183,16 @@ function createDeliveryStore(methods: AgentTriggerDeliveryPersistence): AgentTrifunction publicReceiptStatus(status: AgentTriggerStoredRecord['status'],): AgentTriggerDeliveryReceipt['status'] {- return status === 'batched' ? 'pending' : status;+ if (status === 'batched' || status === 'capability_pending') {+ return 'pending';+ }+ if (status === 'capability_staging') {+ return 'staging';+ }+ if (status === 'capability_dead') {+ return 'dead';+ }+ return status === 'capability_leased' ? 'leased' : status;}function requireDeliveryOrigin(boundOrigin: string | undefined): void {@@ -192,6 +219,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Aconst purgeRecoveryIntervalMs =deps.purgeRecoveryIntervalMs ?? DEFAULT_PURGE_RECOVERY_INTERVAL_MS;const purgeRecoveryLimit = deps.purgeRecoveryLimit ?? DEFAULT_PURGE_RECOVERY_LIMIT;+ const supportsDetachedActionCompletion = deps.supportsDetachedActionCompletion ?? (() => false);if (!Number.isSafeInteger(userDrainTimeoutMs) || userDrainTimeoutMs <= 0) {throw new TypeError('userDrainTimeoutMs must be a positive integer');}@@ -213,8 +241,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Alet stopping = false;const isPrincipalActive = deps.isPrincipalActive;const host: AgentTriggerExecutionHost = createAgentTriggerExecutionHost({- getBaseUrl: () => {- const origin = process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin;+ getBaseUrl: (options) => {+ const origin =+ options?.localOnly === true+ ? boundOrigin+ : (process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin);if (origin == null) {throw new Error('Agent trigger service has not been initialized with a listener address');}@@ -391,7 +422,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}deliveryEngine = createAgentTriggerDeliveryEngine({- store: createDeliveryStore(methods),+ store: createDeliveryStore(methods, supportsDetachedActionCompletion),dispatch: dispatchForActivePrincipal,},deps.deliveryOptions,diff --git a/packages/api/src/agents/triggers/types.ts b/packages/api/src/agents/triggers/types.tsindex ea44518061..40963438de 100644--- a/packages/api/src/agents/triggers/types.ts+++ b/packages/api/src/agents/triggers/types.ts@@ -22,3 +22,17 @@ export interface AgentEventAppliedAction {toolName: string;toolCallId?: string;}++/** Job-store outbox record for terminal detached-action evidence that has not+ * necessarily reached the authoritative delivery row yet. */+export interface AgentEventDetachedTerminalEvidence {+ version: 1;+ deliveryKey: string;+ generationCreatedAt: number;+ taskId: string;+ idempotencyKey: string;+ status: 'succeeded' | 'failed' | 'cancelled';+ result?: string;+ error?: string;+ observedAt: number;+}diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 9c563497fb..fb35be40e9 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -915,6 +915,26 @@ class GenerationJobManagerClass {this.terminalHostActionHandler = handler;}+ /**+ * Event-driven retry for a durable terminal host action whose external+ * evidence arrived after the generation itself became terminal. The exact+ * generation fence prevents a delayed callback from touching a replacement;+ * the periodic recovery sweep remains the restart/replica fallback.+ */+ async retryTerminalHostAction(streamId: string, expectedCreatedAt: number): Promise<boolean> {+ const job = await this.jobStore.getJob(streamId);+ if (+ job?.createdAt !== expectedCreatedAt ||+ job.status === 'running' ||+ job.status === 'requires_action' ||+ job.providerDrained === false ||+ job.terminalHostActionPending !== true+ ) {+ return false;+ }+ return this.runTerminalHostActionHandler(streamId, job);+ }+private get storeLabel(): GenerationJobStore {return this._isRedis ? 'redis' : 'memory';}@@ -2606,6 +2626,10 @@ class GenerationJobManagerClass {// Surface whether the turn was temporary so a resume keeps it non-persisted.isTemporary: jobData.isTemporary,agentEventDeliveryKey: jobData.agentEventDeliveryKey,+ agentEventInvocationKey: jobData.agentEventInvocationKey,+ agentEventInvocationGenerationCreatedAt: jobData.agentEventInvocationGenerationCreatedAt,+ agentEventDetachedActionProducerRequired: jobData.agentEventDetachedActionProducerRequired,+ agentEventDetachedTerminalEvidence: jobData.agentEventDetachedTerminalEvidence,agentEventBindingId: jobData.agentEventBindingId,agentEventExpectedAction: jobData.agentEventExpectedAction,agentEventSuspension: jobData.agentEventSuspension,@@ -6699,6 +6723,34 @@ class GenerationJobManagerClass {await this.jobStore.updateJob(streamId, updates, generationId);}+ /** Stages exact detached terminal evidence in the generation-owned job outbox+ * and verifies the epoch-fenced write before the external result leaves memory. */+ async persistAgentEventDetachedTerminalEvidence(+ streamId: string,+ expectedCreatedAt: number,+ evidence: NonNullable<t.GenerationJobMetadata['agentEventDetachedTerminalEvidence']>,+ ): Promise<boolean> {+ await this.jobStore.updateJob(+ streamId,+ { agentEventDetachedTerminalEvidence: evidence },+ expectedCreatedAt,+ );+ const persisted = await this.jobStore.getJob(streamId);+ const actual = persisted?.agentEventDetachedTerminalEvidence;+ return (+ persisted?.createdAt === expectedCreatedAt &&+ actual?.version === evidence.version &&+ actual.deliveryKey === evidence.deliveryKey &&+ actual.generationCreatedAt === evidence.generationCreatedAt &&+ actual.taskId === evidence.taskId &&+ actual.idempotencyKey === evidence.idempotencyKey &&+ actual.status === evidence.status &&+ actual.result === evidence.result &&+ actual.error === evidence.error &&+ actual.observedAt === evidence.observedAt+ );+ }+/** Records that one exact provider segment has completed every trailing write.* The opaque segment id prevents a paused controller from acknowledging a* later HITL resume that reuses the same generation epoch. */diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.tsindex 3ec4efa182..bc70dba462 100644--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts@@ -339,6 +339,10 @@ describe('RedisJobStore', () => {model: 'test-model',agent_id: 'agent-1',isTemporary: false,+ agentEventDeliveryKey: 'completion-delivery-1',+ agentEventInvocationKey: 'original-delivery-1',+ agentEventInvocationGenerationCreatedAt: 987654,+ agentEventDetachedActionProducerRequired: true,scheduleId: 'schedule-1',scheduledFor: '2026-08-17T12:00:00.000Z',scheduleConfigRevision: 4,@@ -397,6 +401,10 @@ describe('RedisJobStore', () => {parentMessageId: 'parent-1',},responseMessageId: 'response-1',+ agentEventDeliveryKey: 'completion-delivery-1',+ agentEventInvocationKey: 'original-delivery-1',+ agentEventInvocationGenerationCreatedAt: 987654,+ agentEventDetachedActionProducerRequired: true,mcpRequestBody: {messageId: 'response-1',conversationId: 'overridden-conversation',@@ -495,6 +503,35 @@ describe('RedisJobStore', () => {});});+ test('restores detached terminal outbox evidence from Redis', async () => {+ const evidence = {+ version: 1 as const,+ deliveryKey: 'trigger-1',+ generationCreatedAt: 100,+ taskId: 'task-1',+ idempotencyKey: 'a'.repeat(64),+ status: 'succeeded' as const,+ result: 'accepted',+ observedAt: 150,+ };+ const redis = {+ isCluster: true,+ hgetall: jest.fn().mockResolvedValue({+ streamId: 'stream-detached-terminal',+ userId: 'user-1',+ status: 'complete',+ createdAt: '100',+ syncSent: '0',+ agentEventDetachedTerminalEvidence: JSON.stringify(evidence),+ }),+ } as unknown as Cluster;+ const store = new RedisJobStore(redis);++ await expect(store.getJob('stream-detached-terminal')).resolves.toMatchObject({+ agentEventDetachedTerminalEvidence: evidence,+ });+ });+test('atomically resets predecessor state when creating a replacement', async () => {const evalJobCreation = jest.fn()diff --git a/packages/api/src/stream/__tests__/terminalHostAction.spec.ts b/packages/api/src/stream/__tests__/terminalHostAction.spec.tsindex 2bf81b5953..a30785acdc 100644--- a/packages/api/src/stream/__tests__/terminalHostAction.spec.ts+++ b/packages/api/src/stream/__tests__/terminalHostAction.spec.ts@@ -22,6 +22,37 @@ describe('GenerationJobManager terminal host actions', () => {await manager.destroy();});+ it('verifies detached terminal outbox persistence against the exact generation', async () => {+ const streamId = 'conversation-detached-outbox';+ const job = await manager.createJob(streamId, 'user-1', streamId);+ const evidence = {+ version: 1 as const,+ deliveryKey: 'trigger-outbox',+ generationCreatedAt: job.createdAt,+ taskId: 'task-outbox',+ idempotencyKey: 'a'.repeat(64),+ status: 'succeeded' as const,+ result: 'accepted',+ observedAt: job.createdAt + 1,+ };++ await expect(+ manager.persistAgentEventDetachedTerminalEvidence(streamId, job.createdAt, evidence),+ ).resolves.toBe(true);+ await expect(store.getJob(streamId)).resolves.toMatchObject({+ agentEventDetachedTerminalEvidence: evidence,+ });++ const replacement = await store.createJob(streamId, 'user-1', streamId);+ expect(replacement.createdAt).not.toBe(job.createdAt);+ await expect(+ manager.persistAgentEventDetachedTerminalEvidence(streamId, job.createdAt, evidence),+ ).resolves.toBe(false);+ await expect(store.getJob(streamId)).resolves.not.toHaveProperty(+ 'agentEventDetachedTerminalEvidence',+ );+ });+it('runs and acknowledges a generation-fenced host action before terminal cleanup', async () => {const handler = jest.fn().mockResolvedValue(undefined);manager.setTerminalHostActionHandler(handler);@@ -89,6 +120,29 @@ describe('GenerationJobManager terminal host actions', () => {);});+ it('retries an exact pending terminal host action when late evidence arrives', async () => {+ const handler = jest+ .fn()+ .mockRejectedValueOnce(new Error('detached action is still running'))+ .mockResolvedValue(undefined);+ manager.setTerminalHostActionHandler(handler);+ const streamId = 'conversation-late-terminal-evidence';+ const job = await manager.createJob(streamId, 'user-1', streamId, {+ initialMetadata: { agentEventDeliveryKey: 'trigger-late-evidence' },+ });++ await expect(manager.completeJob(streamId, undefined, job.createdAt)).resolves.toBe(true);+ await expect(store.getJob(streamId)).resolves.toMatchObject({+ terminalHostActionPending: true,+ });++ await expect(manager.retryTerminalHostAction(streamId, job.createdAt + 1)).resolves.toBe(false);+ expect(handler).toHaveBeenCalledTimes(1);+ await expect(manager.retryTerminalHostAction(streamId, job.createdAt)).resolves.toBe(true);+ expect(handler).toHaveBeenCalledTimes(2);+ await expect(store.getJob(streamId)).resolves.not.toHaveProperty('terminalHostActionPending');+ });+it('waits for the provider evidence fence and retains a failed tool close', async () => {const handler = jest.fn().mockResolvedValue(undefined);manager.setTerminalHostActionHandler(handler);diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.tsindex 787a8c8831..a987a78350 100644--- a/packages/api/src/stream/implementations/RedisJobStore.ts+++ b/packages/api/src/stream/implementations/RedisJobStore.ts@@ -4738,6 +4738,17 @@ export class RedisJobStore implements IJobStoreV2 {agent_id: data.agent_id || undefined,isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,agentEventDeliveryKey: data.agentEventDeliveryKey || undefined,+ agentEventInvocationKey: data.agentEventInvocationKey || undefined,+ agentEventInvocationGenerationCreatedAt: data.agentEventInvocationGenerationCreatedAt+ ? parseInt(data.agentEventInvocationGenerationCreatedAt, 10)+ : undefined,+ agentEventDetachedActionProducerRequired:+ data.agentEventDetachedActionProducerRequired != null+ ? data.agentEventDetachedActionProducerRequired === '1'+ : undefined,+ agentEventDetachedTerminalEvidence: data.agentEventDetachedTerminalEvidence+ ? JSON.parse(data.agentEventDetachedTerminalEvidence)+ : undefined,agentEventBindingId: data.agentEventBindingId || undefined,agentEventExpectedAction: data.agentEventExpectedAction? JSON.parse(data.agentEventExpectedAction)diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex fde92ad465..c545bf162e 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -5,6 +5,7 @@ import type {UserSubmittedMessageFieldPath,} from 'librechat-data-provider';import type { RunStep, StandardGraph } from '@librechat/agents';+import type { AgentEventDetachedTerminalEvidence } from '~/agents/triggers/types';import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';import type { RecoveredSteerPayload } from '../SteerRecovery';@@ -283,6 +284,14 @@ export interface SerializableJobData {*/isTemporary?: boolean;agentEventDeliveryKey?: string;+ /** Original actor invocation when an internal completion delivery owns this generation. */+ agentEventInvocationKey?: string;+ /** Original actor invocation generation retained across completion HITL resumes. */+ agentEventInvocationGenerationCreatedAt?: number;+ /** This generation must resume on a durable detached-action producer. */+ agentEventDetachedActionProducerRequired?: boolean;+ /** Durable retry payload captured before detached terminal evidence is written to Mongo. */+ agentEventDetachedTerminalEvidence?: AgentEventDetachedTerminalEvidence;/** Trusted actor binding copied from the authenticated delivery envelope. */agentEventBindingId?: string;agentEventExpectedAction?: import('~/agents/triggers/types').AgentTriggerExpectedAction;@@ -408,6 +417,10 @@ export type JobMetadataPatch = Partial<| 'agent_id'| 'isTemporary'| 'agentEventDeliveryKey'+ | 'agentEventInvocationKey'+ | 'agentEventInvocationGenerationCreatedAt'+ | 'agentEventDetachedActionProducerRequired'+ | 'agentEventDetachedTerminalEvidence'| 'agentEventBindingId'| 'agentEventExpectedAction'| 'agentEventSuspension'diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.tsindex 997d29b7ef..ad2f87c41b 100644--- a/packages/api/src/stream/metadata.ts+++ b/packages/api/src/stream/metadata.ts@@ -39,6 +39,20 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): Jif (metadata.agentEventDeliveryKey) {patch.agentEventDeliveryKey = metadata.agentEventDeliveryKey;}+ if (metadata.agentEventInvocationKey) {+ patch.agentEventInvocationKey = metadata.agentEventInvocationKey;+ }+ if (metadata.agentEventInvocationGenerationCreatedAt !== undefined) {+ patch.agentEventInvocationGenerationCreatedAt =+ metadata.agentEventInvocationGenerationCreatedAt;+ }+ if (metadata.agentEventDetachedActionProducerRequired !== undefined) {+ patch.agentEventDetachedActionProducerRequired =+ metadata.agentEventDetachedActionProducerRequired;+ }+ if (metadata.agentEventDetachedTerminalEvidence) {+ patch.agentEventDetachedTerminalEvidence = metadata.agentEventDetachedTerminalEvidence;+ }if (metadata.agentEventBindingId) {patch.agentEventBindingId = metadata.agentEventBindingId;}diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.tsindex b549aa4824..5f9707951a 100644--- a/packages/api/src/types/stream.ts+++ b/packages/api/src/types/stream.ts@@ -1,6 +1,7 @@import type { Agents, UserSubmittedMessageFieldPath } from 'librechat-data-provider';import type { EventEmitter } from 'events';import type {+ AgentEventDetachedTerminalEvidence,AgentEventSuspensionProjection,AgentTriggerExpectedAction,} from '../agents/triggers/types';@@ -47,6 +48,14 @@ export interface GenerationJobMetadata {isTemporary?: boolean;/** Exact durable delivery whose accepted continuation created this generation. */agentEventDeliveryKey?: string;+ /** Original actor invocation when the current mailbox delivery is an internal completion. */+ agentEventInvocationKey?: string;+ /** Original actor invocation generation retained across completion HITL resumes. */+ agentEventInvocationGenerationCreatedAt?: number;+ /** This generation must resume on a durable detached-action producer. */+ agentEventDetachedActionProducerRequired?: boolean;+ /** Durable retry payload captured before detached terminal evidence reaches Mongo. */+ agentEventDetachedTerminalEvidence?: AgentEventDetachedTerminalEvidence;/** Trusted actor binding copied from the authenticated delivery envelope. */agentEventBindingId?: string;/** Optional action evidence contract declared by the authenticated event source. */diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.tsindex edcd562925..a887614093 100644--- a/packages/data-schemas/src/index.ts+++ b/packages/data-schemas/src/index.ts@@ -53,6 +53,7 @@ export {MAX_AGENT_EVENT_ACTOR_SUMMARY_LENGTH,MAX_AGENT_EVENT_ACTOR_TOOL_NAME_LENGTH,} from './types/convo';+export { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from './types/triggerDelivery';export type * from './types';export type * from './methods';export {diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.tsindex 4a4a2e1fb2..815b5c8dec 100644--- a/packages/data-schemas/src/methods/conversation.spec.ts+++ b/packages/data-schemas/src/methods/conversation.spec.ts@@ -4027,6 +4027,9 @@ describe('Conversation Operations', () => {jobCreatedAt: 123,}),).resolves.toEqual({ status: 'stored' });+ await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({+ suspension: { kind: 'human_decision' },+ });await expect(methods.storeAgentEventActorSuspension({...owner,@@ -4056,6 +4059,7 @@ describe('Conversation Operations', () => {methods.storeAgentEventActorSuspension({...owner,suspension: second,+ kind: 'internal_completion',actionId: 'action-second',jobCreatedAt: 123,previous: {@@ -4065,6 +4069,12 @@ describe('Conversation Operations', () => {},}),).resolves.toEqual({ status: 'stored' });+ await expect(methods.getAgentEventActorSnapshot(owner)).resolves.toMatchObject({+ suspension: {+ kind: 'internal_completion',+ suspension: { suspensionId: second.suspensionId },+ },+ });await expect(methods.claimAgentEventActorSuspension({...owner,diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.tsindex de9fb429af..a45d28cddd 100644--- a/packages/data-schemas/src/methods/conversation.ts+++ b/packages/data-schemas/src/methods/conversation.ts@@ -321,6 +321,9 @@ export interface ConversationMethods {conversationId: string;tenantId?: string;suspension: IAgentEventActorSuspensionEvidence;+ kind?: 'human_decision' | 'internal_completion';+ appliedAction?: { toolName: string; toolCallId?: string };+ handlingGenerationCreatedAt?: number;actionId: string;jobCreatedAt: number;/** The segment applied its expected action before publishing this successor pause. */@@ -642,6 +645,9 @@ export function createConversationMethods(conversationId: string;tenantId?: string;suspension: IAgentEventActorSuspensionEvidence;+ kind?: 'human_decision' | 'internal_completion';+ appliedAction?: { toolName: string; toolCallId?: string };+ handlingGenerationCreatedAt?: number;actionId: string;jobCreatedAt: number;invalidateHead?: boolean;@@ -684,15 +690,19 @@ export function createConversationMethods(...activeExpirationFilter<IConversation>(),...predecessor,};+ const suspension = {+ suspension: input.suspension,+ kind: input.kind ?? 'human_decision',+ ...(input.appliedAction == null ? {} : { appliedAction: input.appliedAction }),+ handlingGenerationCreatedAt: input.handlingGenerationCreatedAt ?? input.jobCreatedAt,+ actionId: input.actionId,+ jobCreatedAt: input.jobCreatedAt,+ status: 'pending' as const,+ observedAt: new Date(),+ };const storeUpdate = {$set: {- agentEventActorSuspension: {- suspension: input.suspension,- actionId: input.actionId,- jobCreatedAt: input.jobCreatedAt,- status: 'pending' as const,- observedAt: new Date(),- },+ agentEventActorSuspension: suspension,...(input.invalidateHead === true ? { 'agentEventActor.requiresColdStart': true } : {}),},};@@ -713,13 +723,7 @@ export function createConversationMethods({ ...ownership, agentEventActor: { $exists: false } },{$set: {- agentEventActorSuspension: {- suspension: input.suspension,- actionId: input.actionId,- jobCreatedAt: input.jobCreatedAt,- status: 'pending',- observedAt: new Date(),- },+ agentEventActorSuspension: suspension,},},{ new: false, timestamps: false },diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.tsindex 98fa95da7a..bbb3b5c5ea 100644--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts@@ -15,6 +15,7 @@ import {setAgentEventActorReceiptMetricObserver,type AgentTriggerDeliveryMethods,} from './triggerDelivery';+import { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from '~/types/triggerDelivery';import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';@@ -237,6 +238,142 @@ describe('agent trigger delivery methods', () => {expect(claims.filter((claim) => claim != null)).toHaveLength(1);});+ it('keeps capability-fenced work limited to capable workers through lease recovery', async () => {+ const queued = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+ }),+ );+ expect(queued.delivery).toMatchObject({+ status: 'capability_pending',+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+ });+ const claimInput = {+ workerId: 'capable-worker',+ claimToken: 'capable-claim',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ };++ await expect(+ methods.claimNextAgentTriggerDelivery({+ ...claimInput,+ workerId: 'old-worker',+ claimToken: 'old-claim',+ }),+ ).resolves.toBeNull();+ const capable = await methods.claimNextAgentTriggerDelivery({+ ...claimInput,+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],+ });+ expect(capable).toMatchObject({ status: 'capability_leased' });++ await Delivery.updateOne(+ { _id: capable!.id },+ { $set: { leaseUntil: new Date(START.getTime() - 60_000) } },+ );+ const recoveryNow = new Date(START.getTime() + 60_000);+ await expect(+ methods.claimNextAgentTriggerDelivery({+ workerId: 'old-worker',+ claimToken: 'old-recovery',+ now: recoveryNow,+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),+ }),+ ).resolves.toBeNull();+ const recovered = await methods.claimNextAgentTriggerDelivery({+ workerId: 'capable-worker-2',+ claimToken: 'capable-recovery',+ now: recoveryNow,+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],+ });+ expect(recovered).toMatchObject({ status: 'capability_leased' });++ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: recovered!.id,+ workerId: 'capable-worker-2',+ claimToken: 'capable-recovery',+ now: recoveryNow,+ });+ expect(attempt).toBe(1);+ await expect(+ methods.deadLetterAgentTriggerDelivery({+ id: recovered!.id,+ workerId: 'capable-worker-2',+ claimToken: 'capable-recovery',+ attempt: 1,+ error: transientFailure({ attemptedAt: recoveryNow }),+ settledAt: recoveryNow,+ }),+ ).resolves.toBe(true);+ await expect(Delivery.findById(recovered!.id).lean()).resolves.toMatchObject({+ status: 'capability_dead',+ });++ const [deadLetter] = await methods.getAgentTriggerDeadLetters();+ expect(deadLetter).toMatchObject({ id: recovered!.id, status: 'capability_dead' });+ const requeued = await methods.requeueAgentTriggerDelivery(recovered!.id, recoveryNow);+ expect(requeued).toMatchObject({ status: 'capability_pending' });+ await expect(+ methods.claimNextAgentTriggerDelivery({+ workerId: 'old-worker',+ claimToken: 'old-requeue',+ now: recoveryNow,+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),+ }),+ ).resolves.toBeNull();+ const finalClaim = await methods.claimNextAgentTriggerDelivery({+ workerId: 'capable-worker-3',+ claimToken: 'capable-final',+ now: recoveryNow,+ leaseUntil: new Date(recoveryNow.getTime() + 60_000),+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],+ });+ const finalAttempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: finalClaim!.id,+ workerId: 'capable-worker-3',+ claimToken: 'capable-final',+ now: recoveryNow,+ });+ await expect(+ methods.completeAgentTriggerDelivery({+ id: finalClaim!.id,+ workerId: 'capable-worker-3',+ claimToken: 'capable-final',+ attempt: finalAttempt!,+ result: { status: 'complete' },+ settledAt: recoveryNow,+ }),+ ).resolves.toBe(true);+ });++ it('claims the oldest eligible delivery without capability traffic priority', async () => {+ const ordinary = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ deliveryKey: 'ordinary-older',+ availableAt: new Date(START.getTime() - 2_000),+ }),+ );+ await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ deliveryKey: 'capability-newer',+ availableAt: new Date(START.getTime() - 1_000),+ requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,+ }),+ );++ await expect(+ methods.claimNextAgentTriggerDelivery({+ workerId: 'capable-worker',+ claimToken: 'claim-oldest',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1],+ }),+ ).resolves.toMatchObject({ id: ordinary.delivery.id, status: 'leased' });+ });+it('allocates a replica-stable monotonic sequence within an ordering lane', async () => {const user = new mongoose.Types.ObjectId();const replicas = [@@ -256,6 +393,86 @@ describe('agent trigger delivery methods', () => {);});+ it('does not publish a later requeue through a stale lane reservation', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'stale-publication-generation';+ const originalUpdateOne = Delivery.collection.updateOne.bind(Delivery.collection);+ let releaseStalePublication!: () => void;+ let stalePublicationPaused!: () => void;+ const stalePublicationGate = new Promise<void>((resolve) => {+ releaseStalePublication = resolve;+ });+ const stalePublicationObserved = new Promise<void>((resolve) => {+ stalePublicationPaused = resolve;+ });+ let paused = false;+ const updateSpy = jest+ .spyOn(Delivery.collection, 'updateOne')+ .mockImplementation(async (filter, update, options) => {+ const status = filter.status as { $in?: string[] } | undefined;+ if (+ !paused &&+ filter.orderingKey === orderingKey &&+ status?.$in?.includes('staging') === true &&+ filter.requeueCount === 0+ ) {+ paused = true;+ stalePublicationPaused();+ await stalePublicationGate;+ }+ return originalUpdateOne(filter, update, options);+ });++ try {+ const enqueue = methods.enqueueAgentTriggerDelivery(enqueueInput({ user, orderingKey }));+ await stalePublicationObserved;++ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(1);+ const published = await Delivery.findOne({ orderingKey }).lean();+ expect(published).toMatchObject({ status: 'pending', laneSequence: 1, requeueCount: 0 });++ await originalUpdateOne(+ { _id: published!._id },+ {+ $set: {+ status: 'staging',+ laneSequence: 0,+ requeueCount: 1,+ stagingRecoveryAt: new Date(),+ },+ },+ );+ await LaneSequence.updateOne(+ {+ _id: orderingKey,+ value: 1,+ publisherDeliveryId: { $exists: false },+ },+ {+ $inc: { value: 1 },+ $set: {+ publisherDeliveryId: published!._id,+ publisherRequeueCount: 1,+ publisherStartedAt: new Date(),+ },+ },+ );++ releaseStalePublication();+ await expect(enqueue).resolves.toMatchObject({+ delivery: { status: 'pending', laneSequence: 2, requeueCount: 1 },+ });+ await expect(Delivery.findById(published!._id).lean()).resolves.toMatchObject({+ status: 'pending',+ laneSequence: 2,+ requeueCount: 1,+ });+ } finally {+ releaseStalePublication();+ updateSpy.mockRestore();+ }+ });+it('coalesces a concurrent replica burst behind one claimable root and settles every receipt', async () => {const user = new mongoose.Types.ObjectId();const coalesceUntil = new Date(Date.now() + 60_000);@@ -1758,6 +1975,14 @@ describe('agent trigger delivery methods', () => {await expect(methods.admitAgentEventActorAction(actionAdmission)).resolves.toBe(true);await expect(methods.settleAgentEventActorReceipt(settlement)).resolves.toBe(true);await expect(methods.releaseAgentEventActorAction(actionAdmission)).resolves.toBe(false);+ const settledDelivery = await Delivery.findOne({ deliveryKey: settlement.deliveryKey })+ .select('+actorActionAdmittedAt +actorActionAdmissionId')+ .lean();+ expect(settledDelivery).not.toHaveProperty('actorActionAdmittedAt');+ expect(settledDelivery).not.toHaveProperty('actorActionAdmissionId');+ await expect(+ methods.countActiveAgentTriggerDeliveriesByUser(queued.delivery.user, settlement.settledAt),+ ).resolves.toBe(0);/** Simulate a receipt written by the pre-normalization build after* transport dead-lettered the root. Exact replay must retire that dead* state instead of leaving a non-requeueable dead letter. */@@ -1833,6 +2058,474 @@ describe('agent trigger delivery methods', () => {).resolves.toBeNull();});+ it('persists detached evidence before the started transport response settles', async () => {+ const bindingId = 'binding-detached-leased';+ const conversationId = 'conversation-detached-leased';+ const queued = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ awaitTerminalHandling: true,+ tenantId: undefined,+ envelope: {+ mode: 'continue',+ target: { bindingId },+ event: { source: { id: 'source-key-1', type: 'remote_api_key' } },+ },+ }),+ );+ const claimed = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-detached-leased',+ claimToken: 'claim-detached-leased',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: claimed!.id,+ workerId: 'worker-detached-leased',+ claimToken: 'claim-detached-leased',+ now: START,+ });+ const generationCreatedAt = START.getTime() + 1_000;+ await expect(+ methods.admitAgentEventActorAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ admittedAt: new Date(generationCreatedAt),+ admissionId: 'admission-detached-leased',+ }),+ ).resolves.toBe(true);+ const reservation = await methods.reserveAgentEventActorDetachedAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ turnId: 'response-detached-leased:0',+ invocationId: queued.delivery.deliveryKey,+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-detached-leased',+ reservedAt: new Date(generationCreatedAt + 1_000),+ recoveryAfter: new Date(generationCreatedAt + 61_000),+ });+ expect(reservation.status).toBe('reserved');+ const update = {+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ taskId: reservation.action.taskId,+ idempotencyKey: reservation.action.idempotencyKey,+ observedAt: new Date(generationCreatedAt + 2_000),+ recoveryAfter: new Date(generationCreatedAt + 1_802_000),+ };+ await expect(methods.markAgentEventActorDetachedActionRunning(update)).resolves.toEqual({+ status: 'applied',+ });+ const retryAt = new Date(generationCreatedAt + 3_000);+ await expect(+ methods.retryAgentTriggerDelivery({+ id: claimed!.id,+ workerId: 'worker-detached-leased',+ claimToken: 'claim-detached-leased',+ attempt: attempt!,+ error: transientFailure({ attemptedAt: retryAt }),+ availableAt: retryAt,+ }),+ ).resolves.toBe(true);+ await expect(methods.markAgentEventActorDetachedActionRunning(update)).resolves.toEqual({+ status: 'already_applied',+ });+ const retryClaim = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-detached-retry',+ claimToken: 'claim-detached-retry',+ now: retryAt,+ leaseUntil: new Date(retryAt.getTime() + 60_000),+ });+ const retryAttempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: retryClaim!.id,+ workerId: 'worker-detached-retry',+ claimToken: 'claim-detached-retry',+ now: retryAt,+ });+ await expect(+ methods.deadLetterAgentTriggerDelivery({+ id: retryClaim!.id,+ workerId: 'worker-detached-retry',+ claimToken: 'claim-detached-retry',+ attempt: retryAttempt!,+ error: transientFailure({ attemptedAt: retryAt }),+ settledAt: retryAt,+ }),+ ).resolves.toBe(true);+ await expect(+ methods.settleAgentEventActorDetachedAction({+ ...update,+ status: 'succeeded',+ result: 'accepted',+ observedAt: new Date(generationCreatedAt + 4_000),+ }),+ ).resolves.toEqual({ status: 'applied' });+ await expect(+ methods.getAgentEventActorDetachedAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ }),+ ).resolves.toEqual(expect.objectContaining({ status: 'succeeded', result: 'accepted' }));+ });++ it('reserves a detached retry for the exact leased owner before handling is persisted', async () => {+ const bindingId = 'binding-detached-leased-retry';+ const conversationId = 'conversation-detached-leased-retry';+ const queued = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ awaitTerminalHandling: true,+ tenantId: undefined,+ envelope: {+ mode: 'continue',+ target: { bindingId },+ event: { source: { id: 'source-key-1', type: 'remote_api_key' } },+ },+ }),+ );+ await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-detached-leased-retry',+ claimToken: 'claim-detached-leased-retry',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const generationCreatedAt = START.getTime() + 1_000;+ await expect(+ methods.admitAgentEventActorAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ admittedAt: new Date(generationCreatedAt),+ admissionId: 'admission-detached-leased-retry',+ }),+ ).resolves.toBe(true);+ const first = await methods.reserveAgentEventActorDetachedAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ turnId: 'response-detached-leased-retry:0',+ invocationId: queued.delivery.deliveryKey,+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-detached-leased-retry-1',+ reservedAt: new Date(generationCreatedAt + 1_000),+ recoveryAfter: new Date(generationCreatedAt + 61_000),+ });+ await expect(+ methods.settleAgentEventActorDetachedAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ taskId: first.action.taskId,+ idempotencyKey: first.action.idempotencyKey,+ status: 'failed',+ error: 'definite adapter failure',+ observedAt: new Date(generationCreatedAt + 2_000),+ }),+ ).resolves.toEqual({ status: 'applied' });++ const retryInput = {+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ turnId: 'response-detached-leased-retry:1',+ invocationId: queued.delivery.deliveryKey,+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-detached-leased-retry-2',+ reservedAt: new Date(generationCreatedAt + 3_000),+ recoveryAfter: new Date(generationCreatedAt + 63_000),+ };+ const retry = await methods.reserveAgentEventActorDetachedAction(retryInput);++ expect(retry).toMatchObject({+ status: 'reserved',+ action: {+ launchAttempt: 1,+ toolCallId: 'call-detached-leased-retry-2',+ status: 'reserved',+ },+ });+ await expect(+ methods.settleAgentEventActorDetachedAction({+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ taskId: retry.action.taskId,+ idempotencyKey: retry.action.idempotencyKey,+ status: 'failed',+ error: 'retry adapter failure',+ observedAt: new Date(generationCreatedAt + 4_000),+ }),+ ).resolves.toEqual({ status: 'applied' });+ await expect(+ methods.reserveAgentEventActorDetachedAction({+ ...retryInput,+ reservedAt: new Date(generationCreatedAt + 5_000),+ recoveryAfter: new Date(generationCreatedAt + 65_000),+ }),+ ).resolves.toMatchObject({+ status: 'replay',+ action: {+ taskId: retry.action.taskId,+ idempotencyKey: retry.action.idempotencyKey,+ launchAttempt: 1,+ status: 'failed',+ },+ });+ const stored = await Delivery.findOne({ deliveryKey: queued.delivery.deliveryKey })+ .select('+actorDetachedActionHistory')+ .lean();+ expect(stored).toMatchObject({+ status: 'leased',+ actorDetachedActionHistory: [+ expect.objectContaining({ taskId: first.action.taskId, status: 'failed' }),+ ],+ });+ expect(stored).not.toHaveProperty('handling');+ });++ it('gives one replica the detached expected-action launch reservation', async () => {+ const bindingId = 'binding-detached-1';+ const conversationId = 'conversation-detached-1';+ const queued = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ awaitTerminalHandling: true,+ tenantId: undefined,+ envelope: {+ mode: 'continue',+ target: { bindingId },+ event: { source: { id: 'source-key-1', type: 'remote_api_key' } },+ },+ }),+ );+ const claimed = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-detached-1',+ claimToken: 'claim-detached-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: claimed!.id,+ workerId: 'worker-detached-1',+ claimToken: 'claim-detached-1',+ now: START,+ });+ const generationCreatedAt = START.getTime() + 1_000;+ const admission = {+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ admittedAt: new Date(generationCreatedAt),+ admissionId: 'admission-detached-1',+ };+ await expect(methods.admitAgentEventActorAction(admission)).resolves.toBe(true);+ await methods.completeAgentTriggerDelivery({+ id: claimed!.id,+ workerId: 'worker-detached-1',+ claimToken: 'claim-detached-1',+ attempt: attempt!,+ result: { status: 'started' },+ settledAt: new Date(generationCreatedAt),+ awaitTerminalHandling: true,+ handling: {+ status: 'started',+ conversationId,+ streamId: conversationId,+ generationCreatedAt,+ startedAt: new Date(generationCreatedAt),+ },+ });+ const reservation = {+ deliveryKey: queued.delivery.deliveryKey,+ user: queued.delivery.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ turnId: 'response-detached-1:0',+ invocationId: queued.delivery.deliveryKey,+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-detached-1',+ reservedAt: new Date(generationCreatedAt + 1_000),+ recoveryAfter: new Date(generationCreatedAt + 61_000),+ };++ const contenders = await Promise.all([+ methods.reserveAgentEventActorDetachedAction(reservation),+ methods.reserveAgentEventActorDetachedAction(reservation),+ ]);++ expect(contenders.map((result) => result.status).sort()).toEqual(['replay', 'reserved']);+ expect(contenders[0].action).toEqual(contenders[1].action);+ expect(contenders[0].action).toMatchObject({+ version: 1,+ invocationId: reservation.invocationId,+ expectedToolName: reservation.expectedToolName,+ toolName: reservation.toolName,+ toolCallId: reservation.toolCallId,+ taskId: expect.stringMatching(/^event_actor_[a-f0-9]{64}$/),+ status: 'reserved',+ launchAttempt: 0,+ reservedAt: reservation.reservedAt,+ });+ expect(contenders[0].action.idempotencyKey).toMatch(/^[a-f0-9]{64}$/);+ await expect(methods.reserveAgentEventActorDetachedAction(reservation)).resolves.toEqual({+ status: 'replay',+ action: contenders[0].action,+ });++ const update = {+ deliveryKey: reservation.deliveryKey,+ user: reservation.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ taskId: contenders[0].action.taskId,+ idempotencyKey: contenders[0].action.idempotencyKey,+ observedAt: new Date(generationCreatedAt + 2_000),+ recoveryAfter: new Date(generationCreatedAt + 1_802_000),+ };+ /** The same durable state covers process death both before invoke and in+ * the unobservable invoke->acknowledgement window: neither permits relaunch. */+ await expect(+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({+ ...update,+ observedAt: reservation.recoveryAfter,+ }),+ ).resolves.toEqual({ status: 'applied' });+ await expect(+ methods.settleAgentEventActorDetachedAction({+ ...update,+ status: 'failed',+ error: 'move service unavailable',+ }),+ ).resolves.toEqual({ status: 'applied' });+ await expect(methods.markAgentEventActorDetachedActionRunning(update)).resolves.toEqual({+ status: 'already_applied',+ });+ await expect(+ methods.settleAgentEventActorDetachedAction({+ ...update,+ observedAt: new Date(generationCreatedAt + 5_000),+ status: 'failed',+ error: 'move service unavailable',+ }),+ ).resolves.toEqual({ status: 'already_applied' });+ await expect(+ methods.settleAgentEventActorDetachedAction({+ ...update,+ status: 'succeeded',+ result: 'conflicting result',+ }),+ ).resolves.toEqual({ status: 'conflict' });+ await expect(methods.reserveAgentEventActorDetachedAction(reservation)).resolves.toMatchObject({+ status: 'replay',+ action: { taskId: contenders[0].action.taskId, status: 'failed' },+ });+ const retryReservation = await methods.reserveAgentEventActorDetachedAction({+ ...reservation,+ turnId: 'response-detached-1:1',+ reservedAt: new Date(generationCreatedAt + 3_000),+ recoveryAfter: new Date(generationCreatedAt + 63_000),+ });+ expect(retryReservation).toMatchObject({+ status: 'reserved',+ action: {+ launchAttempt: 1,+ toolCallId: reservation.toolCallId,+ status: 'reserved',+ },+ });+ expect(retryReservation.action.taskId).not.toBe(contenders[0].action.taskId);+ const retryUpdate = {+ ...update,+ taskId: retryReservation.action.taskId,+ idempotencyKey: retryReservation.action.idempotencyKey,+ observedAt: new Date(generationCreatedAt + 4_000),+ };+ await expect(+ methods.markAgentEventActorDetachedActionRunning({+ ...retryUpdate,+ recoveryAfter: new Date(generationCreatedAt + 1_804_000),+ }),+ ).resolves.toEqual({ status: 'applied' });+ await expect(+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({+ ...retryUpdate,+ observedAt: new Date(generationCreatedAt + 1_803_999),+ }),+ ).resolves.toEqual({ status: 'conflict' });+ await expect(+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({+ ...retryUpdate,+ observedAt: new Date(generationCreatedAt + 1_804_000),+ }),+ ).resolves.toEqual({ status: 'applied' });+ await expect(+ methods.markAgentEventActorDetachedActionLaunchIndeterminate({+ ...retryUpdate,+ observedAt: new Date(generationCreatedAt + 1_805_000),+ }),+ ).resolves.toEqual({ status: 'already_applied' });+ await expect(+ methods.settleAgentEventActorDetachedAction({+ ...retryUpdate,+ observedAt: new Date(generationCreatedAt + 1_806_000),+ status: 'succeeded',+ result: 'move accepted',+ }),+ ).resolves.toEqual({ status: 'applied' });+ const stored = await Delivery.findOne({ deliveryKey: reservation.deliveryKey })+ .select('+actorDetachedAction +actorDetachedActionHistory')+ .lean();+ expect(stored?.actorDetachedAction).toMatchObject({+ status: 'succeeded',+ result: 'move accepted',+ launchAttempt: 1,+ });+ expect(stored?.actorDetachedActionHistory).toEqual([+ expect.objectContaining({+ taskId: contenders[0].action.taskId,+ status: 'failed',+ error: 'move service unavailable',+ }),+ ]);+ await expect(+ methods.getAgentEventActorDetachedAction({+ deliveryKey: reservation.deliveryKey,+ user: reservation.user,+ bindingId,+ conversationId,+ generationCreatedAt,+ }),+ ).resolves.toMatchObject({+ taskId: retryReservation.action.taskId,+ status: 'succeeded',+ result: 'move accepted',+ });+ });+it('serializes no-action settlement against renewed action admission', async () => {const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput({@@ -2649,10 +3342,11 @@ describe('agent trigger delivery methods', () => {});});- it('counts only live leases while an account deletion drains', async () => {+ it('counts live leases and detached launch authority while an account deletion drains', async () => {const user = new mongoose.Types.ObjectId();- await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));- await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ const leased = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ const detached = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ const admitted = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));await methods.claimNextAgentTriggerDelivery({workerId: 'worker-1',claimToken: 'claim-1',@@ -2660,7 +3354,73 @@ describe('agent trigger delivery methods', () => {leaseUntil: new Date(START.getTime() + 60_000),});- expect(await methods.countActiveAgentTriggerDeliveriesByUser(user, START)).toBe(1);+ await Delivery.updateOne(+ { _id: detached.delivery.id },+ {+ $set: {+ status: 'succeeded',+ actorDetachedAction: {+ version: 1,+ invocationId: detached.delivery.deliveryKey,+ expectedToolName: 'submit_move',+ toolName: 'submit_move_mcp_chess',+ toolCallId: 'call-delete-drain',+ turnId: 'response-delete-drain:0',+ taskId: `event_actor_${'d'.repeat(64)}`,+ idempotencyKey: 'd'.repeat(64),+ launchAttempt: 0,+ status: 'running',+ reservedAt: START,+ observedAt: START,+ recoveryAfter: new Date(START.getTime() + 60_000),+ launchedAt: START,+ },+ },+ },+ );+ await Delivery.updateOne(+ { _id: admitted.delivery.id },+ {+ $set: {+ status: 'succeeded',+ actorActionAdmittedAt: START,+ actorActionAdmissionId: 'admission-delete-drain',+ },+ },+ );++ expect(leased.delivery.id).not.toBe(detached.delivery.id);+ expect(await methods.countActiveAgentTriggerDeliveriesByUser(user, START)).toBe(3);+ expect(+ await methods.countActiveAgentTriggerDeliveriesByUser(+ user,+ new Date(START.getTime() + 60_001),+ ),+ ).toBe(2);+ await Delivery.updateOne(+ { _id: detached.delivery.id },+ { $set: { 'actorDetachedAction.status': 'launch_indeterminate' } },+ );+ expect(+ await methods.countActiveAgentTriggerDeliveriesByUser(+ user,+ new Date(START.getTime() + 60_001),+ ),+ ).toBe(2);+ await Delivery.updateOne(+ { _id: detached.delivery.id },+ { $set: { 'actorDetachedAction.status': 'succeeded' } },+ );+ expect(+ await methods.countActiveAgentTriggerDeliveriesByUser(+ user,+ new Date(START.getTime() + 60_001),+ ),+ ).toBe(1);+ await Delivery.updateOne(+ { _id: admitted.delivery.id },+ { $unset: { actorActionAdmittedAt: 1, actorActionAdmissionId: 1 } },+ );expect(await methods.countActiveAgentTriggerDeliveriesByUser(user,@@ -2669,6 +3429,67 @@ describe('agent trigger delivery methods', () => {).toBe(0);});+ it('serializes account deletion against action admission on each delivery row', async () => {+ const user = new mongoose.Types.ObjectId();+ const fenceStartedAt = new Date(START);+ await User.create({+ _id: user,+ email: '[email protected]',+ provider: 'local',+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ const first = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ user,+ envelope: { mode: 'continue', target: { bindingId: 'binding-purge' } },+ }),+ );+ const second = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({+ user,+ envelope: { mode: 'continue', target: { bindingId: 'binding-purge' } },+ }),+ );+ await Delivery.updateMany({ user }, { $set: { status: 'succeeded' } });+ const admission = {+ deliveryKey: first.delivery.deliveryKey,+ user,+ tenantId: 'tenant-1',+ bindingId: 'binding-purge',+ conversationId: 'conversation-purge',+ admittedAt: START,+ admissionId: 'admission-before-purge',+ };++ await expect(methods.admitAgentEventActorAction(admission)).resolves.toBe(true);+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);+ await expect(+ methods.countActiveAgentTriggerDeliveriesByUser(user, new Date(START.getTime() + 60_001)),+ ).resolves.toBe(1);+ await expect(+ methods.admitAgentEventActorAction({+ ...admission,+ deliveryKey: second.delivery.deliveryKey,+ admissionId: 'admission-after-purge',+ }),+ ).resolves.toBe(false);+ await expect(+ Delivery.countDocuments({ user, actorActionAdmissionClosedAt: fenceStartedAt }),+ ).resolves.toBe(2);++ await expect(methods.cancelAgentTriggerUserPurge(user, fenceStartedAt)).resolves.toBe(true);+ await expect(+ Delivery.countDocuments({ user, actorActionAdmissionClosedAt: { $exists: true } }),+ ).resolves.toBe(0);+ await expect(+ methods.admitAgentEventActorAction({+ ...admission,+ deliveryKey: second.delivery.deliveryKey,+ admissionId: 'admission-after-cancel',+ }),+ ).resolves.toBe(true);+ });+it('deletes all queued payloads for an erased user', async () => {const user = new mongoose.Types.ObjectId();const fenceStartedAt = new Date(START);@@ -2758,6 +3579,9 @@ describe('agent trigger delivery methods', () => {await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(0);expect(await Delivery.countDocuments({ user })).toBe(1);+ expect(+ await Delivery.countDocuments({ user, actorActionAdmissionClosedAt: { $exists: true } }),+ ).toBe(0);expect(await UserPurge.countDocuments({ _id: user })).toBe(0);});diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.tsindex d300d62191..c507276d1a 100644--- a/packages/data-schemas/src/methods/triggerDelivery.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.ts@@ -1,5 +1,7 @@+import { createHash } from 'node:crypto';import type { Model, Types } from 'mongoose';import type {+ AgentEventActorDetachedAction,AgentTriggerDeliveryClaim,AgentEventActorReceipt,AgentTriggerDeliveryFailure,@@ -14,6 +16,7 @@ import type {IAgentTriggerUserPurge,IAgentTriggerUserPurgeDocument,} from '~/types/triggerDelivery';+import { AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 } from '~/types/triggerDelivery';import { createIndexesWithRetry } from '~/utils/retry';import logger from '~/config/winston';@@ -29,6 +32,10 @@ const HISTORY_LIMIT = 64;const MAX_BATCH_SIZE = 8;const MAX_BATCH_BYTES = 512 * 1024;+function isStagingStatus(status: IAgentTriggerDelivery['status']): boolean {+ return status === 'staging' || status === 'capability_staging';+}+type DuplicateKeyError = { code?: number };export type AgentEventActorReceiptMetric = {@@ -75,6 +82,7 @@ export interface EnqueueAgentTriggerDeliveryInput {coalesceFrom?: Date;coalesceUntil?: Date;awaitTerminalHandling?: boolean;+ requiredWorkerCapability?: string;}export interface AgentTriggerDeliveryFence {@@ -118,6 +126,42 @@ export interface AgentEventActorActionAdmissionInput extends GetAgentEventActorRadmissionId: string;}+/** Durable transition receipt. Storage unavailability remains an exception;+ * callers never collapse it into a conflict or a negative acknowledgement. */+export interface AgentEventActorDetachedTransitionResult {+ status: 'applied' | 'already_applied' | 'conflict';+}++export interface ReserveAgentEventActorDetachedActionInput extends GetAgentEventActorReceiptInput {+ generationCreatedAt: number;+ turnId: string;+ invocationId: string;+ expectedToolName: string;+ toolName: string;+ toolCallId: string;+ reservedAt: Date;+ recoveryAfter: Date;+}++export interface UpdateAgentEventActorDetachedActionInput extends GetAgentEventActorReceiptInput {+ generationCreatedAt: number;+ taskId: string;+ idempotencyKey: string;+ observedAt: Date;+}++export interface MarkAgentEventActorDetachedActionRunningInput+ extends UpdateAgentEventActorDetachedActionInput {+ recoveryAfter: Date;+}++export interface SettleAgentEventActorDetachedActionInput+ extends UpdateAgentEventActorDetachedActionInput {+ status: 'succeeded' | 'failed' | 'cancelled';+ result?: string;+ error?: string;+}+export interface GetAgentEventActorReceiptInput {deliveryKey: string;user: string | Types.ObjectId;@@ -145,6 +189,7 @@ export interface AgentTriggerDeliveryMethods {claimToken: string;now: Date;leaseUntil: Date;+ workerCapabilities?: string[];}) => Promise<AgentTriggerDeliveryClaim | null>;findEarlierAgentTriggerDelivery: (delivery: Pick<AgentTriggerDeliveryRecord, 'orderingKey' | 'laneSequence'>,@@ -181,6 +226,24 @@ export interface AgentTriggerDeliveryMethods {hasAgentEventActorActionAdmission: (input: AgentEventActorActionAdmissionInput,) => Promise<boolean>;+ reserveAgentEventActorDetachedAction: (+ input: ReserveAgentEventActorDetachedActionInput,+ ) => Promise<{+ status: 'reserved' | 'replay' | 'conflict';+ action: AgentEventActorDetachedAction;+ }>;+ markAgentEventActorDetachedActionRunning: (+ input: MarkAgentEventActorDetachedActionRunningInput,+ ) => Promise<AgentEventActorDetachedTransitionResult>;+ markAgentEventActorDetachedActionLaunchIndeterminate: (+ input: UpdateAgentEventActorDetachedActionInput,+ ) => Promise<AgentEventActorDetachedTransitionResult>;+ settleAgentEventActorDetachedAction: (+ input: SettleAgentEventActorDetachedActionInput,+ ) => Promise<AgentEventActorDetachedTransitionResult>;+ getAgentEventActorDetachedAction: (+ input: GetAgentEventActorReceiptInput & { generationCreatedAt: number },+ ) => Promise<AgentEventActorDetachedAction | null>;settleAgentEventActorReceipt: (input: SettleAgentEventActorReceiptInput) => Promise<boolean>;getAgentEventActorReceipt: (input: GetAgentEventActorReceiptInput,@@ -260,6 +323,9 @@ function toRecord(delivery: IAgentTriggerDelivery): AgentTriggerDeliveryRecord {availableAt: delivery.availableAt,createdAt: delivery.createdAt,...(delivery.tenantId != null && { tenantId: delivery.tenantId }),+ ...(delivery.requiredWorkerCapability != null && {+ requiredWorkerCapability: delivery.requiredWorkerCapability,+ }),...(delivery.leaseBy != null && { leaseBy: delivery.leaseBy }),...(delivery.leaseUntil != null && { leaseUntil: delivery.leaseUntil }),...(delivery.claimToken != null && { claimToken: delivery.claimToken }),@@ -295,7 +361,7 @@ function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliv}const record = toRecord(delivery);if (- record.status !== 'leased' ||+ !['leased', 'capability_leased'].includes(record.status) ||record.claimToken == null ||record.leaseBy == null ||record.leaseUntil == null@@ -335,6 +401,9 @@ export function createAgentTriggerDeliveryMethods(_id: lane._id,value: lane.value,publisherDeliveryId,+ ...(lane.publisherRequeueCount == null+ ? { publisherRequeueCount: { $exists: false } }+ : { publisherRequeueCount: lane.publisherRequeueCount }),};if (lane.value === 1) {const deleted = await LaneSequence().deleteOne(publisherFence);@@ -343,7 +412,7 @@ export function createAgentTriggerDeliveryMethods(const released = await LaneSequence().updateOne(publisherFence, {$inc: { value: -1 },$set: { cleanupRequestedAt: new Date() },- $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },+ $unset: { publisherDeliveryId: 1, publisherRequeueCount: 1, publisherStartedAt: 1 },});return released.modifiedCount === 1;}@@ -358,6 +427,26 @@ export function createAgentTriggerDeliveryMethods(}const staged = await Delivery().findById(publisherDeliveryId).lean<IAgentTriggerDelivery>();+ const publicationGeneration = lane.publisherRequeueCount ?? staged?.requeueCount ?? 0;+ if (lane.publisherRequeueCount == null) {+ /** Upgrade a publisher acquired by a pre-generation binary before+ * touching its delivery. The exact lane fence prevents a stale helper+ * from adopting the requeue generation of a later publisher. */+ const adopted = await LaneSequence().updateOne(+ {+ _id: lane._id,+ value: lane.value,+ publisherDeliveryId,+ publisherRequeueCount: { $exists: false },+ },+ { $set: { publisherRequeueCount: publicationGeneration } },+ { timestamps: false },+ );+ if (adopted.modifiedCount !== 1) {+ return false;+ }+ }+ const publicationLane = { ...lane, publisherRequeueCount: publicationGeneration };let batchRoot: IAgentTriggerDelivery | null = null;if (staged?._id != null &&@@ -432,11 +521,22 @@ export function createAgentTriggerDeliveryMethods(batchRoot.awaitTerminalHandling = true;}}+ let publishedStatus: IAgentTriggerDelivery['status'] = 'pending';+ if (batchRoot != null) {+ publishedStatus = 'batched';+ } else if (staged?.requiredWorkerCapability != null) {+ publishedStatus = 'capability_pending';+ }const published = await Delivery().updateOne(- { _id: publisherDeliveryId, orderingKey: lane._id, status: 'staging' },+ {+ _id: publisherDeliveryId,+ orderingKey: lane._id,+ status: { $in: ['staging', 'capability_staging'] },+ requeueCount: publicationGeneration,+ },{$set: {- status: batchRoot == null ? 'pending' : 'batched',+ status: publishedStatus,laneSequence: lane.value,...(batchRoot?._id != null && {batchRootId: batchRoot._id,@@ -450,19 +550,27 @@ export function createAgentTriggerDeliveryMethods(if (published.modifiedCount === 0) {let current = await Delivery().findById(publisherDeliveryId)- .select('orderingKey laneSequence status batchRootId')+ .select('orderingKey laneSequence status batchRootId requeueCount').lean<- Pick<IAgentTriggerDelivery, 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId'>+ Pick<+ IAgentTriggerDelivery,+ 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId' | 'requeueCount'+ >>();if (batchRoot?._id != null &¤t != null &¤t.orderingKey === lane._id &&- current.status !== 'staging' &&+ !isStagingStatus(current.status) &¤t.laneSequence !== lane.value) {await Delivery().updateOne(- { _id: publisherDeliveryId, orderingKey: lane._id, status: current.status },+ {+ _id: publisherDeliveryId,+ orderingKey: lane._id,+ status: current.status,+ requeueCount: publicationGeneration,+ },{$set: {laneSequence: lane.value,@@ -473,17 +581,24 @@ export function createAgentTriggerDeliveryMethods();current = await Delivery().findById(publisherDeliveryId)- .select('orderingKey laneSequence status batchRootId')+ .select('orderingKey laneSequence status batchRootId requeueCount').lean<- Pick<IAgentTriggerDelivery, 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId'>+ Pick<+ IAgentTriggerDelivery,+ 'orderingKey' | 'laneSequence' | 'status' | 'batchRootId' | 'requeueCount'+ >>();}publicationCommitted =current != null &¤t.orderingKey === lane._id &&- current.status !== 'staging' &&+ !isStagingStatus(current.status) &&+ current.requeueCount === publicationGeneration &¤t.laneSequence === lane.value;- if (current?.orderingKey === lane._id && current.status === 'staging') {+ if (current?.orderingKey === lane._id && isStagingStatus(current.status)) {+ if (current.requeueCount !== publicationGeneration) {+ return abandonLanePublisher(publicationLane);+ }throw new Error('Failed to publish the reserved agent trigger delivery');}}@@ -492,11 +607,12 @@ export function createAgentTriggerDeliveryMethods(_id: lane._id,value: lane.value,publisherDeliveryId,+ publisherRequeueCount: publicationGeneration,};if (publicationCommitted) {const released = await LaneSequence().updateOne(publisherFence, {$set: { tailDeliveryId: publisherDeliveryId },- $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },+ $unset: { publisherDeliveryId: 1, publisherRequeueCount: 1, publisherStartedAt: 1 },});return published.modifiedCount === 1 || released.modifiedCount === 1;}@@ -505,7 +621,7 @@ export function createAgentTriggerDeliveryMethods(// reservation was acquired (or was removed by account cleanup). No later// reservation can exist while this fence is held, so the unused sequence// can be rolled back without creating a gap or regressing the lane tail.- return abandonLanePublisher(lane);+ return abandonLanePublisher(publicationLane);}/** A dead batch root owns requeue before any constituent is reset. Staging@@ -572,7 +688,7 @@ export function createAgentTriggerDeliveryMethods(if (current == null) {throw new Error('Staged agent trigger delivery disappeared before publication');}- if (current.status !== 'staging') {+ if (!isStagingStatus(current.status)) {return current;}if ((await UserPurge().exists({ _id: current.user })) != null) {@@ -593,7 +709,7 @@ export function createAgentTriggerDeliveryMethods(// the oldest visible row first lets any replica repair a writer that// stopped in that gap without allowing a later enqueue to overtake it.const next = await Delivery()- .findOne({ orderingKey, status: 'staging' })+ .findOne({ orderingKey, status: { $in: ['staging', 'capability_staging'] } })// updatedAt is the current staging admission time: initial enqueue// sets it on insert, while explicit requeue refreshes it so an old// dead letter is admitted behind staging work that already exists.@@ -612,6 +728,7 @@ export function createAgentTriggerDeliveryMethods($inc: { value: 1 },$set: {publisherDeliveryId: next._id,+ publisherRequeueCount: next.requeueCount ?? 0,publisherStartedAt: new Date(),},$unset: { cleanupRequestedAt: 1 },@@ -647,6 +764,15 @@ export function createAgentTriggerDeliveryMethods(async function enqueueAgentTriggerDelivery(input: EnqueueAgentTriggerDeliveryInput,): Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }> {+ if (+ input.requiredWorkerCapability != null &&+ input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1+ ) {+ throw new TypeError('Agent trigger delivery requires an unsupported worker capability');+ }+ if (input.requiredWorkerCapability != null && input.coalesceKey != null) {+ throw new TypeError('Capability-fenced agent trigger deliveries cannot be batched');+ }let staged: IAgentTriggerDelivery;let replayed = false;try {@@ -660,7 +786,7 @@ export function createAgentTriggerDeliveryMethods(batchMemberIds: [],}),laneSequence: 0,- status: 'staging',+ status: input.requiredWorkerCapability == null ? 'staging' : 'capability_staging',attempts: 0,requeueCount: 0,stagingRecoveryAt: new Date(),@@ -679,7 +805,7 @@ export function createAgentTriggerDeliveryMethods(if (existing.fingerprint !== input.fingerprint) {throw new AgentTriggerDeliveryConflictError(input.deliveryKey);}- if (existing.status !== 'staging') {+ if (!isStagingStatus(existing.status)) {return { delivery: toRecord(existing), replayed: true };}staged = existing;@@ -716,6 +842,9 @@ export function createAgentTriggerDeliveryMethods(_id: lane._id,value: lane.value,publisherDeliveryId: lane.publisherDeliveryId,+ ...(lane.publisherRequeueCount == null+ ? { publisherRequeueCount: { $exists: false } }+ : { publisherRequeueCount: lane.publisherRequeueCount }),publisherStartedAt: lane.publisherStartedAt,},{ $set: { publisherStartedAt: recoveryCursor } },@@ -736,7 +865,10 @@ export function createAgentTriggerDeliveryMethods(/** Pre-cursor replicas can leave staging rows outside the sparse recovery* index. Backfill them in bounded batches before using the indexed scan. */const legacyStaged = await Delivery()- .find({ status: 'staging', stagingRecoveryAt: { $exists: false } })+ .find({+ status: { $in: ['staging', 'capability_staging'] },+ stagingRecoveryAt: { $exists: false },+ }).sort({ updatedAt: 1, _id: 1 }).limit(remaining).lean<IAgentTriggerDelivery[]>();@@ -744,7 +876,11 @@ export function createAgentTriggerDeliveryMethods(for (const delivery of legacyStaged) {if (delivery._id != null) {await Delivery().updateOne(- { _id: delivery._id, status: 'staging', stagingRecoveryAt: { $exists: false } },+ {+ _id: delivery._id,+ status: { $in: ['staging', 'capability_staging'] },+ stagingRecoveryAt: { $exists: false },+ },{ $set: { stagingRecoveryAt: legacyCursor } },{ timestamps: false },);@@ -758,7 +894,7 @@ export function createAgentTriggerDeliveryMethods(remaining > 0? await Delivery().find({- status: 'staging',+ status: { $in: ['staging', 'capability_staging'] },stagingRecoveryAt: { $exists: true },...(legacyIds.length > 0 && { _id: { $nin: legacyIds } }),})@@ -778,7 +914,7 @@ export function createAgentTriggerDeliveryMethods(await Delivery().updateOne({_id: delivery._id,- status: 'staging',+ status: { $in: ['staging', 'capability_staging'] },stagingRecoveryAt: delivery.stagingRecoveryAt,},{ $set: { stagingRecoveryAt: stagingRecoveryCursor } },@@ -788,7 +924,7 @@ export function createAgentTriggerDeliveryMethods(for (const delivery of staged) {try {const published = await publishStagedDelivery(delivery);- if (published.status !== 'staging') {+ if (!isStagingStatus(published.status)) {recovered += 1;} else {await rotateStagingRecovery(delivery);@@ -815,7 +951,21 @@ export function createAgentTriggerDeliveryMethods(const stillRetained = await Delivery().exists({orderingKey,$or: [- { status: { $in: ['staging', 'batched', 'pending', 'leased', 'dead'] } },+ {+ status: {+ $in: [+ 'staging',+ 'capability_staging',+ 'batched',+ 'pending',+ 'capability_pending',+ 'leased',+ 'capability_leased',+ 'capability_dead',+ 'dead',+ ],+ },+ },{status: 'succeeded',batchRootId: { $exists: false },@@ -908,25 +1058,56 @@ export function createAgentTriggerDeliveryMethods(claimToken: string;now: Date;leaseUntil: Date;+ workerCapabilities?: string[];}): Promise<AgentTriggerDeliveryClaim | null> {const expiredBefore = new Date(input.now.getTime() - LEASE_SKEW_MARGIN_MS);+ const workerCapabilities = [...new Set(input.workerCapabilities ?? [])];+ if (workerCapabilities.some((value) => value.length === 0 || value.length > 128)) {+ throw new TypeError('Agent trigger worker capability is invalid');+ }+ const capabilityConditions =+ workerCapabilities.length === 0+ ? []+ : [+ {+ requiredWorkerCapability: { $in: workerCapabilities },+ status: 'capability_pending',+ availableAt: { $lte: input.now },+ },+ {+ requiredWorkerCapability: { $in: workerCapabilities },+ status: 'capability_leased',+ leaseUntil: { $lte: expiredBefore },+ },+ ];const claimed = await Delivery().findOneAndUpdate({$or: [{ status: 'pending', availableAt: { $lte: input.now } },{ status: 'leased', leaseUntil: { $lte: expiredBefore } },+ ...capabilityConditions,],},- {- $set: {- status: 'leased',- leaseBy: input.workerId,- leaseUntil: input.leaseUntil,- claimToken: input.claimToken,+ [+ {+ $set: {+ status: {+ $cond: [+ { $in: ['$status', ['capability_pending', 'capability_leased']] },+ 'capability_leased',+ 'leased',+ ],+ },+ leaseBy: input.workerId,+ leaseUntil: input.leaseUntil,+ claimToken: input.claimToken,+ settledAt: '$$REMOVE',+ expiresAt: '$$REMOVE',+ updatedAt: input.now,+ },},- $unset: { settledAt: 1, expiresAt: 1 },- },+ ],{ new: true, sort: { availableAt: 1, createdAt: 1, _id: 1 } },).lean<IAgentTriggerDelivery>();@@ -941,7 +1122,11 @@ export function createAgentTriggerDeliveryMethods(orderingKey: delivery.orderingKey,laneSequence: { $lt: delivery.laneSequence },$or: [- { status: { $in: ['pending', 'leased'] } },+ {+ status: {+ $in: ['pending', 'capability_pending', 'leased', 'capability_leased'],+ },+ },{status: 'succeeded',batchRootId: { $exists: false },@@ -1205,14 +1390,27 @@ export function createAgentTriggerDeliveryMethods(const fence = (input: AgentTriggerDeliveryFence) => ({_id: input.id,- status: 'leased',+ status: { $in: ['leased', 'capability_leased'] },leaseBy: input.workerId,claimToken: input.claimToken,});+ const capabilityFence = (input: AgentTriggerDeliveryFence) => ({+ ...fence(input),+ status: 'capability_leased',+ requiredWorkerCapability: { $exists: true },+ });+async function releaseAgentTriggerDelivery(input: AgentTriggerDeliveryFence & { availableAt: Date },): Promise<boolean> {+ const capabilityResult = await Delivery().updateOne(capabilityFence(input), {+ $set: { status: 'capability_pending', availableAt: input.availableAt },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },+ });+ if (capabilityResult.modifiedCount === 1) {+ return true;+ }const result = await Delivery().updateOne(fence(input), {$set: { status: 'pending', availableAt: input.availableAt },$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },@@ -1241,14 +1439,25 @@ export function createAgentTriggerDeliveryMethods(if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) {throw new TypeError('attempt must be a positive integer');}- const result = await Delivery().updateOne(- { ...fence(input), attempts: input.attempt },+ const update = {+ $inc: { attempts: -1 },+ $set: { availableAt: input.availableAt },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },+ };+ const capabilityResult = await Delivery().updateOne(+ { ...capabilityFence(input), attempts: input.attempt },{- $inc: { attempts: -1 },- $set: { status: 'pending', availableAt: input.availableAt },- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },+ ...update,+ $set: { ...update.$set, status: 'capability_pending' },},);+ if (capabilityResult.modifiedCount === 1) {+ return true;+ }+ const result = await Delivery().updateOne(+ { ...fence(input), attempts: input.attempt },+ { ...update, $set: { ...update.$set, status: 'pending' } },+ );return result.modifiedCount === 1;}@@ -1491,6 +1700,10 @@ export function createAgentTriggerDeliveryMethods(},$unset: {lastError: 1,+ ...(input.requiresActionAdmission === true && {+ actorActionAdmittedAt: 1,+ actorActionAdmissionId: 1,+ }),...(applied && { 'handling.error': 1 }),...(!applied && { 'handling.action': 1 }),},@@ -1534,7 +1747,13 @@ export function createAgentTriggerDeliveryMethods($max: {expiresAt: new Date(actorReceipt.settledAt.getTime() + SUCCESS_RETENTION_MS),},- $unset: { lastError: 1 },+ $unset: {+ lastError: 1,+ ...(input.requiresActionAdmission === true && {+ actorActionAdmittedAt: 1,+ actorActionAdmissionId: 1,+ }),+ },},{ new: true },)@@ -1598,6 +1817,7 @@ export function createAgentTriggerDeliveryMethods('envelope.target.bindingId': input.bindingId,actorReceipt: { $exists: false },actorActionAdmittedAt: { $exists: false },+ actorActionAdmissionClosedAt: { $exists: false },$or: [{ handling: { $exists: false } },{@@ -1689,6 +1909,333 @@ export function createAgentTriggerDeliveryMethods(: null;}+ /** Atomically gives one replica launch authority for an exact detached expected action. */+ async function reserveAgentEventActorDetachedAction(+ input: ReserveAgentEventActorDetachedActionInput,+ ): Promise<{+ status: 'reserved' | 'replay' | 'conflict';+ action: AgentEventActorDetachedAction;+ }> {+ const requiredIdentity = [+ String(input.user),+ input.deliveryKey,+ input.bindingId,+ input.conversationId,+ String(input.generationCreatedAt),+ input.turnId,+ input.invocationId,+ input.expectedToolName,+ input.toolName,+ input.toolCallId,+ ];+ const identity = [String(input.user), input.tenantId ?? '', ...requiredIdentity.slice(1)];+ if (+ requiredIdentity.some((value) => value.length === 0) ||+ !Number.isSafeInteger(input.generationCreatedAt) ||+ input.generationCreatedAt < 0 ||+ input.turnId.length > 512 ||+ Number.isNaN(input.reservedAt.getTime()) ||+ Number.isNaN(input.recoveryAfter.getTime()) ||+ input.recoveryAfter <= input.reservedAt+ ) {+ throw new TypeError('Detached event actor action identity is invalid');+ }+ const buildAction = (launchAttempt: number): AgentEventActorDetachedAction => {+ const idempotencyKey = createHash('sha256')+ .update([...identity, String(launchAttempt)].join('\0'))+ .digest('hex');+ return {+ version: 1,+ invocationId: input.invocationId,+ expectedToolName: input.expectedToolName,+ toolName: input.toolName,+ toolCallId: input.toolCallId,+ turnId: input.turnId,+ /** Public task handles must not disclose the adapter idempotency secret. */+ taskId: `event_actor_${createHash('sha256')+ .update('librechat:event-actor:task:v1\0')+ .update(idempotencyKey)+ .digest('hex')}`,+ idempotencyKey,+ launchAttempt,+ status: 'reserved',+ reservedAt: input.reservedAt,+ observedAt: input.reservedAt,+ recoveryAfter: input.recoveryAfter,+ };+ };+ const action = buildAction(0);+ const tenantScope =+ input.tenantId == null ? { tenantId: { $exists: false } } : { tenantId: input.tenantId };+ const reservationOwner = {+ $or: [+ {+ status: { $in: ['leased', 'capability_leased'] },+ handling: { $exists: false },+ },+ {+ status: { $in: ['succeeded', 'dead'] },+ 'handling.status': 'started',+ 'handling.conversationId': input.conversationId,+ 'handling.generationCreatedAt': input.generationCreatedAt,+ },+ ],+ };+ const reserved = await Delivery()+ .findOneAndUpdate(+ {+ deliveryKey: input.deliveryKey,+ user: input.user,+ ...tenantScope,+ 'envelope.target.bindingId': input.bindingId,+ ...reservationOwner,+ actorReceipt: { $exists: false },+ actorActionAdmittedAt: { $exists: true },+ actorDetachedAction: { $exists: false },+ },+ { $set: { actorDetachedAction: action } },+ { new: true },+ )+ .select('+actorDetachedAction')+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();+ if (reserved?.actorDetachedAction != null) {+ return { status: 'reserved', action: reserved.actorDetachedAction };+ }+ const existing = await Delivery()+ .findOne({+ deliveryKey: input.deliveryKey,+ user: input.user,+ ...tenantScope,+ 'envelope.target.bindingId': input.bindingId,+ ...reservationOwner,+ })+ .select('+actorDetachedAction')+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();+ if (existing?.actorDetachedAction == null) {+ throw new Error('Detached event actor action reservation owner is unavailable');+ }+ const sameTurn = existing.actorDetachedAction.turnId === input.turnId;+ const sameLogicalAction =+ sameTurn &&+ existing.actorDetachedAction.invocationId === input.invocationId &&+ existing.actorDetachedAction.expectedToolName === input.expectedToolName &&+ existing.actorDetachedAction.toolName === input.toolName &&+ existing.actorDetachedAction.toolCallId === input.toolCallId;+ if (sameLogicalAction) {+ return { status: 'replay', action: existing.actorDetachedAction };+ }+ if (sameTurn) {+ return { status: 'conflict', action: existing.actorDetachedAction };+ }+ if (existing.actorDetachedAction.idempotencyKey === action.idempotencyKey) {+ return { status: 'replay', action: existing.actorDetachedAction };+ }+ if (+ !['failed', 'cancelled'].includes(existing.actorDetachedAction.status) ||+ existing.actorDetachedAction.launchAttempt >= 15+ ) {+ return { status: 'conflict', action: existing.actorDetachedAction };+ }+ const retryAction = buildAction(existing.actorDetachedAction.launchAttempt + 1);+ const retried = await Delivery()+ .findOneAndUpdate(+ {+ deliveryKey: input.deliveryKey,+ user: input.user,+ ...tenantScope,+ 'envelope.target.bindingId': input.bindingId,+ ...reservationOwner,+ actorReceipt: { $exists: false },+ actorActionAdmittedAt: { $exists: true },+ 'actorDetachedAction.taskId': existing.actorDetachedAction.taskId,+ 'actorDetachedAction.idempotencyKey': existing.actorDetachedAction.idempotencyKey,+ 'actorDetachedAction.status': existing.actorDetachedAction.status,+ },+ {+ $set: { actorDetachedAction: retryAction },+ $push: {+ actorDetachedActionHistory: {+ $each: [existing.actorDetachedAction],+ $slice: -8,+ },+ },+ },+ { new: true },+ )+ .select('+actorDetachedAction')+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();+ if (retried?.actorDetachedAction != null) {+ return { status: 'reserved', action: retried.actorDetachedAction };+ }+ const winner = await Delivery()+ .findOne({+ deliveryKey: input.deliveryKey,+ user: input.user,+ ...tenantScope,+ 'envelope.target.bindingId': input.bindingId,+ ...reservationOwner,+ })+ .select('+actorDetachedAction')+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();+ if (winner?.actorDetachedAction == null) {+ throw new Error('Detached event actor retry reservation owner is unavailable');+ }+ return {+ status:+ winner.actorDetachedAction.idempotencyKey === retryAction.idempotencyKey+ ? 'replay'+ : 'conflict',+ action: winner.actorDetachedAction,+ };+ }++ function detachedActionScope(input: UpdateAgentEventActorDetachedActionInput) {+ return {+ deliveryKey: input.deliveryKey,+ user: input.user,+ ...(input.tenantId == null ? { tenantId: { $exists: false } } : { tenantId: input.tenantId }),+ 'envelope.target.bindingId': input.bindingId,+ actorReceipt: { $exists: false },+ actorActionAdmittedAt: { $exists: true },+ 'actorDetachedAction.taskId': input.taskId,+ 'actorDetachedAction.idempotencyKey': input.idempotencyKey,+ };+ }++ /** Acknowledges launch without regressing terminal evidence from a fast completion. */+ async function markAgentEventActorDetachedActionRunning(+ input: MarkAgentEventActorDetachedActionRunningInput,+ ): Promise<AgentEventActorDetachedTransitionResult> {+ if (+ Number.isNaN(input.observedAt.getTime()) ||+ Number.isNaN(input.recoveryAfter.getTime()) ||+ input.recoveryAfter <= input.observedAt+ ) {+ throw new TypeError('Detached event actor running lease is invalid');+ }+ const scope = detachedActionScope(input);+ const running = await Delivery().updateOne(+ { ...scope, 'actorDetachedAction.status': 'reserved' },+ {+ $set: {+ 'actorDetachedAction.status': 'running',+ 'actorDetachedAction.launchedAt': input.observedAt,+ 'actorDetachedAction.observedAt': input.observedAt,+ 'actorDetachedAction.recoveryAfter': input.recoveryAfter,+ },+ },+ );+ if (running.modifiedCount === 1) {+ return { status: 'applied' };+ }+ const alreadyApplied =+ (await Delivery().exists({+ ...scope,+ 'actorDetachedAction.status': {+ $in: ['running', 'succeeded', 'failed', 'cancelled'],+ },+ })) != null;+ return { status: alreadyApplied ? 'already_applied' : 'conflict' };+ }++ /** Converts an expired executor lease into durable uncertainty. This state+ * deliberately cannot be retried; only exact terminal proof may close it. */+ async function markAgentEventActorDetachedActionLaunchIndeterminate(+ input: UpdateAgentEventActorDetachedActionInput,+ ): Promise<AgentEventActorDetachedTransitionResult> {+ if (Number.isNaN(input.observedAt.getTime())) {+ throw new TypeError('observedAt must be a valid date');+ }+ const scope = detachedActionScope(input);+ const marked = await Delivery().updateOne(+ {+ ...scope,+ 'actorDetachedAction.status': { $in: ['reserved', 'running'] },+ 'actorDetachedAction.recoveryAfter': { $lte: input.observedAt },+ },+ {+ $set: {+ 'actorDetachedAction.status': 'launch_indeterminate',+ 'actorDetachedAction.observedAt': input.observedAt,+ },+ },+ );+ if (marked.modifiedCount === 1) {+ return { status: 'applied' };+ }+ const alreadyApplied =+ (await Delivery().exists({+ ...scope,+ 'actorDetachedAction.status': 'launch_indeterminate',+ })) != null;+ return { status: alreadyApplied ? 'already_applied' : 'conflict' };+ }++ /** Persists exact terminal evidence once; identical callbacks replay safely. */+ async function settleAgentEventActorDetachedAction(+ input: SettleAgentEventActorDetachedActionInput,+ ): Promise<AgentEventActorDetachedTransitionResult> {+ if (+ Number.isNaN(input.observedAt.getTime()) ||+ (input.result != null && input.result.length > 32_768) ||+ (input.error != null && input.error.length > 2_048)+ ) {+ throw new TypeError('Detached event actor terminal evidence is invalid');+ }+ if (input.status === 'succeeded' ? input.error != null : input.result != null) {+ throw new TypeError('Detached event actor terminal evidence conflicts with its status');+ }+ const terminal = {+ 'actorDetachedAction.status': input.status,+ 'actorDetachedAction.settledAt': input.observedAt,+ 'actorDetachedAction.observedAt': input.observedAt,+ ...(input.result == null ? {} : { 'actorDetachedAction.result': input.result }),+ ...(input.error == null ? {} : { 'actorDetachedAction.error': input.error }),+ };+ const scope = detachedActionScope(input);+ const settled = await Delivery().updateOne(+ {+ ...scope,+ 'actorDetachedAction.status': {+ $in: ['reserved', 'running', 'launch_indeterminate'],+ },+ },+ { $set: terminal },+ );+ if (settled.modifiedCount === 1) {+ return { status: 'applied' };+ }+ /** Callback observation time is transport metadata, not terminal identity.+ * A retry may arrive later; match the exact durable outcome while retaining+ * the first settlement timestamp. */+ const alreadyApplied =+ (await Delivery().exists({+ ...scope,+ 'actorDetachedAction.status': input.status,+ 'actorDetachedAction.result': input.result == null ? { $exists: false } : input.result,+ 'actorDetachedAction.error': input.error == null ? { $exists: false } : input.error,+ })) != null;+ return { status: alreadyApplied ? 'already_applied' : 'conflict' };+ }++ async function getAgentEventActorDetachedAction(+ input: GetAgentEventActorReceiptInput & { generationCreatedAt: number },+ ): Promise<AgentEventActorDetachedAction | null> {+ const delivery = await Delivery()+ .findOne({+ deliveryKey: input.deliveryKey,+ user: input.user,+ ...(input.tenantId == null+ ? { tenantId: { $exists: false } }+ : { tenantId: input.tenantId }),+ 'envelope.target.bindingId': input.bindingId,+ actorDetachedAction: { $exists: true },+ })+ .select('+actorDetachedAction')+ .lean<Pick<IAgentTriggerDelivery, 'actorDetachedAction'>>();+ return delivery?.actorDetachedAction ?? null;+ }+async function getAgentEventActorReceipt(input: GetAgentEventActorReceiptInput,): Promise<AgentEventActorReceipt | null> {@@ -1853,14 +2400,14 @@ export function createAgentTriggerDeliveryMethods(]),Delivery().countDocuments({ actorReceipt: { $exists: true }, expiresAt: { $lte: now } }),Delivery().countDocuments({- status: 'pending',+ status: { $in: ['pending', 'capability_pending'] },attempts: { $gt: 0 },'envelope.target.bindingId': { $exists: true },'handling.status': 'started',actorReceipt: { $exists: false },}),Delivery().countDocuments({- status: 'dead',+ status: { $in: ['dead', 'capability_dead'] },'envelope.target.bindingId': { $exists: true },'handling.status': 'started',actorReceipt: { $exists: false },@@ -1887,8 +2434,8 @@ export function createAgentTriggerDeliveryMethods(},): Promise<boolean> {const error = normalizeFailure(input.error);- const result = await Delivery().updateOne(fence(input), {- $set: { status: 'pending', availableAt: input.availableAt, lastError: error },+ const retryUpdate = (status: 'pending' | 'capability_pending') => ({+ $set: { status, availableAt: input.availableAt, lastError: error },$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, settledAt: 1, expiresAt: 1 },$push: {history: {@@ -1905,6 +2452,14 @@ export function createAgentTriggerDeliveryMethods(},},});+ const capabilityResult = await Delivery().updateOne(+ capabilityFence(input),+ retryUpdate('capability_pending'),+ );+ if (capabilityResult.modifiedCount === 1) {+ return true;+ }+ const result = await Delivery().updateOne(fence(input), retryUpdate('pending'));return result.modifiedCount === 1;}@@ -1916,29 +2471,33 @@ export function createAgentTriggerDeliveryMethods(},): Promise<boolean> {const error = normalizeFailure(input.error);- const dead = await Delivery()- .findOneAndUpdate(- fence(input),- {- $set: { status: 'dead', settledAt: input.settledAt, lastError: error },- $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },- $push: {- history: {- $each: [- {- attempt: input.attempt,- outcome: 'dead',- at: input.settledAt,- workerId: input.workerId,- error,- },- ],- $slice: -HISTORY_LIMIT,+ const deadUpdate = (status: 'dead' | 'capability_dead') => ({+ $set: { status, settledAt: input.settledAt, lastError: error },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },+ $push: {+ history: {+ $each: [+ {+ attempt: input.attempt,+ outcome: 'dead' as const,+ at: input.settledAt,+ workerId: input.workerId,+ error,},- },+ ],+ $slice: -HISTORY_LIMIT,},- { new: true },- )+ },+ });+ const capabilityDead = await Delivery().updateOne(+ capabilityFence(input),+ deadUpdate('capability_dead'),+ );+ if (capabilityDead.modifiedCount === 1) {+ return true;+ }+ const dead = await Delivery()+ .findOneAndUpdate(fence(input), deadUpdate('dead'), { new: true }).select('_id orderingKey batchMemberIds batchMembersSettledAt requeueCount').lean<Pick<@@ -2039,7 +2598,7 @@ export function createAgentTriggerDeliveryMethods(}const boundedLimit = Math.min(limit, MAX_DEAD_LETTER_LIMIT);const deliveries = await Delivery()- .find({ status: 'dead', batchRootId: { $exists: false } })+ .find({ status: { $in: ['dead', 'capability_dead'] }, batchRootId: { $exists: false } }).sort({ updatedAt: -1, _id: -1 }).limit(boundedLimit).lean<IAgentTriggerDelivery[]>();@@ -2053,7 +2612,7 @@ export function createAgentTriggerDeliveryMethods(const candidate = await Delivery().findOne({_id: id,- status: 'dead',+ status: { $in: ['dead', 'capability_dead'] },batchRootId: { $exists: false },actorReceipt: { $exists: false },actorActionAdmittedAt: { $exists: false },@@ -2071,7 +2630,7 @@ export function createAgentTriggerDeliveryMethods(.findOneAndUpdate({_id: candidate._id,- status: 'dead',+ status: candidate.status,batchRootId: { $exists: false },actorReceipt: { $exists: false },actorActionAdmittedAt: { $exists: false },@@ -2080,7 +2639,7 @@ export function createAgentTriggerDeliveryMethods(},{$set: {- status: 'staging',+ status: candidate.requiredWorkerCapability == null ? 'staging' : 'capability_staging',laneSequence: 0,attempts: 0,availableAt,@@ -2116,7 +2675,24 @@ export function createAgentTriggerDeliveryMethods(user: string | Types.ObjectId,now: Date,): Promise<number> {- return Delivery().countDocuments({ user, status: 'leased', leaseUntil: { $gt: now } });+ return Delivery().countDocuments({+ user,+ $or: [+ {+ actorActionAdmittedAt: { $exists: true },+ actorReceipt: { $exists: false },+ },+ {+ status: { $in: ['leased', 'capability_leased'] },+ leaseUntil: { $gt: now },+ },+ {+ 'actorDetachedAction.status': {+ $in: ['reserved', 'running', 'launch_indeterminate'],+ },+ },+ ],+ });}function requireValidFence(fenceStartedAt: Date): void {@@ -2149,6 +2725,10 @@ export function createAgentTriggerDeliveryMethods(},{ upsert: true },);+ await Delivery().updateMany(+ { user },+ { $set: { actorActionAdmissionClosedAt: fenceStartedAt } },+ );}/** Disarms only the pre-commit deletion attempt that owns this marker. */@@ -2164,8 +2744,12 @@ export function createAgentTriggerDeliveryMethods(if (ownsLiveFence == null) {return false;}- const result = await UserPurge().deleteOne({ _id: user, fenceStartedAt });- return result.deletedCount === 1;+ await Delivery().updateMany(+ { user, actorActionAdmissionClosedAt: fenceStartedAt },+ { $unset: { actorActionAdmissionClosedAt: 1 } },+ );+ await UserPurge().deleteOne({ _id: user, fenceStartedAt });+ return true;}/** Recovers cleanup markers whose users are gone; active-user markers are never destructive. */@@ -2195,6 +2779,10 @@ export function createAgentTriggerDeliveryMethods();continue;}+ await Delivery().updateMany(+ { user: marker._id, actorActionAdmissionClosedAt: marker.fenceStartedAt },+ { $unset: { actorActionAdmissionClosedAt: 1 } },+ );await UserPurge().deleteOne({ _id: marker._id, fenceStartedAt: marker.fenceStartedAt });continue;}@@ -2231,6 +2819,11 @@ export function createAgentTriggerDeliveryMethods(releaseAgentEventActorAction,getAgentEventActorActionAdmission,hasAgentEventActorActionAdmission,+ reserveAgentEventActorDetachedAction,+ markAgentEventActorDetachedActionRunning,+ markAgentEventActorDetachedActionLaunchIndeterminate,+ settleAgentEventActorDetachedAction,+ getAgentEventActorDetachedAction,settleAgentEventActorReceipt,getAgentEventActorReceipt,backfillAgentEventActorReceipt,diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.tsindex 186fa8fa5b..6ce9caf8d4 100644--- a/packages/data-schemas/src/schema/convo.ts+++ b/packages/data-schemas/src/schema/convo.ts@@ -234,6 +234,20 @@ const convoSchema: Schema<IConversation> = new Schema(agentEventActorSuspension: {type: {suspension: { type: Schema.Types.Mixed, required: true },+ kind: {+ type: String,+ enum: ['human_decision', 'internal_completion'],+ default: 'human_decision',+ },+ appliedAction: {+ type: {+ toolName: { type: String, required: true },+ toolCallId: { type: String, default: undefined },+ },+ _id: false,+ default: undefined,+ },+ handlingGenerationCreatedAt: { type: Number, min: 0, default: undefined },actionId: { type: String, required: true },jobCreatedAt: { type: Number, required: true },status: { type: String, enum: ['pending', 'claimed', 'closed'], required: true },diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.tsindex 7d23845fb7..c3dc166c40 100644--- a/packages/data-schemas/src/schema/triggerDelivery.ts+++ b/packages/data-schemas/src/schema/triggerDelivery.ts@@ -85,6 +85,33 @@ const actorReceiptSchema = new Schema({ _id: false },);+const actorDetachedActionSchema = new Schema(+ {+ version: { type: Number, enum: [1], required: true },+ invocationId: { type: String, required: true, maxlength: 128 },+ expectedToolName: { type: String, required: true, maxlength: 256 },+ toolName: { type: String, required: true, maxlength: 256 },+ toolCallId: { type: String, required: true, maxlength: 256 },+ turnId: { type: String, required: true, maxlength: 512 },+ taskId: { type: String, required: true, maxlength: 128 },+ idempotencyKey: { type: String, required: true, minlength: 64, maxlength: 64 },+ launchAttempt: { type: Number, min: 0, max: 15, required: true },+ status: {+ type: String,+ enum: ['reserved', 'running', 'launch_indeterminate', 'succeeded', 'failed', 'cancelled'],+ required: true,+ },+ reservedAt: { type: Date, required: true },+ observedAt: { type: Date, required: true },+ recoveryAfter: { type: Date, required: true },+ launchedAt: { type: Date },+ settledAt: { type: Date },+ result: { type: String, maxlength: 32_768 },+ error: { type: String, maxlength: 2_048 },+ },+ { _id: false },+);+const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema({deliveryKey: { type: String, required: true, maxlength: 128 },@@ -97,10 +124,22 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(tenantId: { type: String, index: true },status: {type: String,- enum: ['staging', 'batched', 'pending', 'leased', 'succeeded', 'dead'],+ enum: [+ 'staging',+ 'capability_staging',+ 'batched',+ 'pending',+ 'capability_pending',+ 'leased',+ 'capability_leased',+ 'succeeded',+ 'capability_dead',+ 'dead',+ ],required: true,default: 'pending',},+ requiredWorkerCapability: { type: String, maxlength: 128 },attempts: { type: Number, required: true, default: 0, min: 0 },availableAt: { type: Date, required: true },envelopeBytes: { type: Number, min: 0 },@@ -119,8 +158,15 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(awaitTerminalHandling: { type: Boolean },handling: { type: handlingSchema },actorReceipt: { type: actorReceiptSchema, select: false },+ actorDetachedAction: { type: actorDetachedActionSchema, select: false },+ actorDetachedActionHistory: {+ type: [actorDetachedActionSchema],+ default: undefined,+ select: false,+ },actorActionAdmittedAt: { type: Date, select: false },actorActionAdmissionId: { type: String, maxlength: 64, select: false },+ actorActionAdmissionClosedAt: { type: Date, select: false },leaseBy: { type: String },leaseUntil: { type: Date },claimToken: { type: String },@@ -155,6 +201,7 @@ triggerDeliverySchema.index();triggerDeliverySchema.index({ status: 1, updatedAt: -1 });triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });+triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });// Only successful rows receive expiresAt. Dead letters remain available untildiff --git a/packages/data-schemas/src/schema/triggerLaneSequence.ts b/packages/data-schemas/src/schema/triggerLaneSequence.tsindex 5a6bbb1369..bcb92fd8bc 100644--- a/packages/data-schemas/src/schema/triggerLaneSequence.ts+++ b/packages/data-schemas/src/schema/triggerLaneSequence.ts@@ -9,6 +9,7 @@ const triggerLaneSequenceSchema: Schema<IAgentTriggerLaneSequenceDocument> = newtenantId: { type: String, index: true },tailDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },publisherDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },+ publisherRequeueCount: { type: Number, min: 0 },publisherStartedAt: { type: Date },cleanupRequestedAt: { type: Date, index: true },},diff --git a/packages/data-schemas/src/types/convo.ts b/packages/data-schemas/src/types/convo.tsindex 0b8a94a009..bfe8601078 100644--- a/packages/data-schemas/src/types/convo.ts+++ b/packages/data-schemas/src/types/convo.ts@@ -146,6 +146,15 @@ export interface IAgentEventActorSuspensionEvidence {*/export interface IAgentEventActorSuspension {suspension: IAgentEventActorSuspensionEvidence;+ /** Host-side reason for suspension. Missing legacy values are human decisions. */+ kind?: 'human_decision' | 'internal_completion';+ /** Expected-action evidence already applied before a later re-pause. */+ appliedAction?: {+ toolName: string;+ toolCallId?: string;+ };+ /** Original delivery-handling generation retained across resumed generations. */+ handlingGenerationCreatedAt?: number;actionId: string;jobCreatedAt: number;status: 'pending' | 'claimed' | 'closed';diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.tsindex 450582629b..671e6c63c7 100644--- a/packages/data-schemas/src/types/triggerDelivery.ts+++ b/packages/data-schemas/src/types/triggerDelivery.ts@@ -3,11 +3,16 @@ import type { IAgentEventActorReconciliation } from './convo';export type AgentTriggerDeliveryStatus =| 'staging'+ | 'capability_staging'| 'batched'| 'pending'+ | 'capability_pending'| 'leased'+ | 'capability_leased'| 'succeeded'+ | 'capability_dead'| 'dead';+export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1';export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';export interface AgentTriggerHandlingState {@@ -36,6 +41,30 @@ export interface AgentEventActorReceipt {settledAt: Date;}+/** Private launch authority for one delivery-owned detached expected action. */+export interface AgentEventActorDetachedAction {+ version: 1;+ invocationId: string;+ expectedToolName: string;+ toolName: string;+ toolCallId: string;+ /** Stable graph-turn identity, independent of retry allocation. */+ turnId: string;+ taskId: string;+ idempotencyKey: string;+ launchAttempt: number;+ status: 'reserved' | 'running' | 'launch_indeterminate' | 'succeeded' | 'failed' | 'cancelled';+ reservedAt: Date;+ observedAt: Date;+ /** A recovery fence, not relaunch authority. Expiry only permits the exact+ * launch to be marked indeterminate while late terminal proof remains valid. */+ recoveryAfter: Date;+ launchedAt?: Date;+ settledAt?: Date;+ result?: string;+ error?: string;+}+export interface AgentTriggerDeliveryFailure {code: string;message: string;@@ -64,6 +93,8 @@ export interface IAgentTriggerDelivery {user: Types.ObjectId;tenantId?: string;status: AgentTriggerDeliveryStatus;+ /** Keeps a delivery invisible to pre-capability workers during rolling deploys. */+ requiredWorkerCapability?: string;attempts: number;availableAt: Date;envelopeBytes?: number;@@ -81,11 +112,17 @@ export interface IAgentTriggerDelivery {awaitTerminalHandling?: boolean;handling?: AgentTriggerHandlingState;actorReceipt?: AgentEventActorReceipt;+ /** Durable launch identity; excluded from ordinary delivery reads. */+ actorDetachedAction?: AgentEventActorDetachedAction;+ /** Bounded audit trail for terminal attempts replaced by an explicit retry. */+ actorDetachedActionHistory?: AgentEventActorDetachedAction[];/** Delivery-owned serialization point acquired exactly once before an event* actor may invoke an external action. */actorActionAdmittedAt?: Date;/** Attempt identity that fences admission takeover and release. */actorActionAdmissionId?: string;+ /** Account-deletion fence that atomically closes action admission on this delivery. */+ actorActionAdmissionClosedAt?: Date;leaseBy?: string;leaseUntil?: Date;claimToken?: string;@@ -136,6 +173,8 @@ export interface IAgentTriggerLaneSequence {tailDeliveryId?: Types.ObjectId;/** Delivery currently owning the serialized sequence/publication step. */publisherDeliveryId?: Types.ObjectId;+ /** Requeue generation captured when this publisher reservation was acquired. */+ publisherRequeueCount?: number;publisherStartedAt?: Date;cleanupRequestedAt?: Date;createdAt?: Date;@@ -163,7 +202,7 @@ export interface AgentTriggerDeliveryClaim extends AgentTriggerDeliveryRecord {claimToken: string;leaseBy: string;leaseUntil: Date;- status: 'leased';+ status: 'leased' | 'capability_leased';}export interface AgentTriggerOrderingBlock {