Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
📬 feat: Add Durable Agent Trigger Delivery (#14925)
Danny Avilaaa35cd42b149
diff --git a/.env.example b/.env.exampleindex 0613685b58..301ab8adb6 100644--- a/.env.example+++ b/.env.example@@ -1204,6 +1204,19 @@ OPENWEATHER_API_KEY=# or# COHERE_API_KEY=your_cohere_api_key+#===========================#+# Agent Trigger Delivery #+#===========================#++# Base URL used by trusted in-process event producers to dispatch agent fires and steers.+# Defaults to this process's bound listener. Set only when internal trigger admission must+# traverse another trusted HTTP origin, such as a TLS front door.+# AGENT_TRIGGERS_SELF_URL=http://127.0.0.1:3080++# Trusted event adapters enqueue through the shared durable trigger service. Mongo-backed+# leases make its workers safe across replicas; successful delivery records expire after+# 90 days, while dead letters remain available for explicit operator requeue.+#======================## MCP Configuration ##======================#diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.jsindex 8cc32e4a85..b8fecb4f8e 100644--- a/api/server/controllers/UserController.js+++ b/api/server/controllers/UserController.js@@ -3,6 +3,7 @@ const { logger, getTenantId, webSearchKeys } = require('@librechat/data-schemas'const {getNewS3URL,needsRefresh,+ GenerationJobManager,MCPOAuthHandler,MCPTokenStorage,getAppConfigOptionsFromUser,@@ -24,6 +25,12 @@ const { verifyEmail, resendVerificationEmail } = require('~/server/services/Authconst { getMCPManager, getFlowStateManager, getMCPServersRegistry } = require('~/config');const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');const { processDeleteRequest } = require('~/server/services/Files/process');+const {+ drainAgentTriggerDeliveriesForUser,+ prepareAgentTriggerUserPurge,+ cancelAgentTriggerUserPurge,+ purgeAgentTriggerDeliveriesForUser,+} = require('~/server/services/Agents/triggers');const { getAppConfig } = require('~/server/services/Config');const { getLogStores } = require('~/cache');const db = require('~/models');@@ -351,6 +358,8 @@ const updateUserPluginsController = async (req, res) => {const deleteUserController = async (req, res) => {const { user } = req;+ let triggerDeletionFence;+ let userDeleted = false;try {const existingUser = await db.getUserById(@@ -369,6 +378,31 @@ const deleteUserController = async (req, res) => {}}+ // Block new trigger admissions across replicas while preserving the user+ // principal so a transient cleanup failure remains retryable.+ triggerDeletionFence = new Date();+ const fenceState = await db.beginAgentTriggerUserDeletion(user.id, triggerDeletionFence);+ if (fenceState === 'in_progress') {+ triggerDeletionFence = undefined;+ throw new Error('Agent trigger account deletion is already in progress');+ }+ if (fenceState === 'missing') {+ triggerDeletionFence = undefined;+ }+ if (triggerDeletionFence != null) {+ await prepareAgentTriggerUserPurge(user.id, triggerDeletionFence, user.tenantId);+ }+ await drainAgentTriggerDeliveriesForUser(user.id);+ const activeAgentRuns = await GenerationJobManager.getCleanupBlockingJobIdsForUser(+ user.id,+ user.tenantId,+ );+ await Promise.all(+ activeAgentRuns.map((streamId) =>+ GenerationJobManager.abortJob(streamId, { awaitProviderDrain: true }),+ ),+ );+await db.deleteMessages({ user: user.id });await db.deleteAllUserSessions({ userId: user.id });await db.deleteTransactions({ user: user.id });@@ -394,7 +428,6 @@ const deleteUserController = async (req, res) => {logger.error('[deleteUserController] Error deleting user convos, likely no convos', error);}await deleteUserPluginAuth(user.id, null, true);- await db.deleteUserById(user.id);await deleteAllSharedLinksWithCleanup(user.id);await deleteUserFiles(req);await db.deleteFiles(null, user.id);@@ -411,9 +444,30 @@ const deleteUserController = async (req, res) => {await db.deleteTokens({ userId: user.id });await db.removeUserFromAllGroups(user.id);await db.deleteAclEntries({ principalId: user._id });+ const deleteResult = await db.deleteUserById(user.id);+ if (deleteResult.deletedCount !== 1) {+ throw new Error('User disappeared before account deletion could commit');+ }+ userDeleted = true;+ await purgeAgentTriggerDeliveriesForUser(user.id);logger.info(`User deleted account. Email: ${user.email} ID: ${user.id}`);res.status(200).send({ message: 'User deleted' });} catch (err) {+ if (triggerDeletionFence != null && !userDeleted) {+ try {+ await cancelAgentTriggerUserPurge(user.id, triggerDeletionFence);+ } catch (purgeFenceError) {+ logger.error(+ '[deleteUserController] Failed to disarm trigger purge recovery',+ purgeFenceError,+ );+ }+ try {+ await db.cancelAgentTriggerUserDeletion(user.id, triggerDeletionFence);+ } catch (fenceError) {+ logger.error('[deleteUserController] Failed to release trigger deletion fence', fenceError);+ }+ }logger.error('[deleteUserController]', err);return res.status(500).json({ message: 'Something went wrong.' });}diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.jsindex 1466dce291..b21fe904e3 100644--- a/api/server/controllers/UserController.spec.js+++ b/api/server/controllers/UserController.spec.js@@ -1,6 +1,13 @@const mongoose = require('mongoose');const { MongoMemoryServer } = require('mongodb-memory-server');+const mockGetActiveJobIdsForUser = jest.fn().mockResolvedValue([]);+const mockAbortJob = jest.fn().mockResolvedValue({ success: true });+const mockDrainAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undefined);+const mockPrepareAgentTriggerUserPurge = jest.fn().mockResolvedValue(undefined);+const mockCancelAgentTriggerUserPurge = jest.fn().mockResolvedValue(true);+const mockPurgeAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undefined);+jest.mock('@librechat/data-schemas', () => {const actual = jest.requireActual('@librechat/data-schemas');return {@@ -26,7 +33,9 @@ jest.mock('~/models', () => {deleteAclEntries: jest.fn().mockResolvedValue(undefined),updateUserPlugins: jest.fn(),deleteAssistants: jest.fn().mockResolvedValue(undefined),- deleteUserById: jest.fn().mockResolvedValue(undefined),+ deleteUserById: jest.fn().mockResolvedValue({ deletedCount: 1 }),+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),+ cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),deleteUserPrompts: jest.fn().mockResolvedValue(undefined),deleteUserSkills: jest.fn().mockResolvedValue(undefined),deleteMessages: jest.fn().mockResolvedValue(undefined),@@ -73,6 +82,17 @@ jest.mock('@librechat/api', () => ({...jest.requireActual('@librechat/api'),needsRefresh: jest.fn(),getNewS3URL: jest.fn(),+ GenerationJobManager: {+ getCleanupBlockingJobIdsForUser: (...args) => mockGetActiveJobIdsForUser(...args),+ abortJob: (...args) => mockAbortJob(...args),+ },+}));++jest.mock('~/server/services/Agents/triggers', () => ({+ drainAgentTriggerDeliveriesForUser: (...args) => mockDrainAgentTriggerDeliveriesForUser(...args),+ prepareAgentTriggerUserPurge: (...args) => mockPrepareAgentTriggerUserPurge(...args),+ cancelAgentTriggerUserPurge: (...args) => mockCancelAgentTriggerUserPurge(...args),+ purgeAgentTriggerDeliveriesForUser: (...args) => mockPurgeAgentTriggerDeliveriesForUser(...args),}));jest.mock('~/server/services/Files/process', () => ({@@ -117,7 +137,14 @@ const {verifyEmailController,} = require('./UserController');const { Group } = require('~/db/models');-const { deleteConvos, acceptTerms } = require('~/models');+const {+ deleteConvos,+ acceptTerms,+ deleteUserById,+ deleteMessages,+ beginAgentTriggerUserDeletion,+ cancelAgentTriggerUserDeletion,+} = require('~/models');const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService');describe('verifyEmailController', () => {@@ -321,6 +348,77 @@ describe('deleteUserController', () => {expect(mockRes.status).toHaveBeenCalledWith(200);expect(mockRes.send).toHaveBeenCalledWith({ message: 'User deleted' });+ expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(userId.toString(), expect.any(Date));+ expect(mockPrepareAgentTriggerUserPurge).toHaveBeenCalledWith(+ userId.toString(),+ expect.any(Date),+ undefined,+ );+ expect(mockDrainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());+ expect(beginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(+ mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0],+ );+ expect(mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0]).toBeLessThan(+ mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0],+ );+ expect(mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0]).toBeLessThan(+ deleteMessages.mock.invocationCallOrder[0],+ );+ expect(deleteMessages.mock.invocationCallOrder[0]).toBeLessThan(+ deleteUserById.mock.invocationCallOrder[0],+ );+ expect(deleteUserById.mock.invocationCallOrder[0]).toBeLessThan(+ mockPurgeAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0],+ );+ expect(mockPurgeAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());+ expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();+ expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();+ });++ it('aborts generations admitted before the deletion fence before erasing messages', async () => {+ const userId = new mongoose.Types.ObjectId();+ mockGetActiveJobIdsForUser.mockResolvedValueOnce(['stream-1', 'stream-2']);+ const req = {+ user: {+ id: userId.toString(),+ _id: userId,+ email: '[email protected]',+ tenantId: 'tenant-1',+ },+ };++ await deleteUserController(req, mockRes);++ expect(mockGetActiveJobIdsForUser).toHaveBeenCalledWith(userId.toString(), 'tenant-1');+ expect(mockAbortJob).toHaveBeenCalledWith('stream-1', { awaitProviderDrain: true });+ expect(mockAbortJob).toHaveBeenCalledWith('stream-2', { awaitProviderDrain: true });+ expect(mockAbortJob.mock.invocationCallOrder[1]).toBeLessThan(+ deleteMessages.mock.invocationCallOrder[0],+ );+ });++ it('fails closed and releases deletion fences when a provider cannot confirm drain', async () => {+ const userId = new mongoose.Types.ObjectId();+ const userIdString = userId.toString();+ mockGetActiveJobIdsForUser.mockResolvedValueOnce(['stream-still-writing']);+ mockAbortJob.mockRejectedValueOnce(new Error('provider drain timed out'));+ const req = {+ user: {+ id: userIdString,+ _id: userId,+ email: '[email protected]',+ tenantId: 'tenant-1',+ },+ };++ await deleteUserController(req, mockRes);++ expect(mockRes.status).toHaveBeenCalledWith(500);+ expect(deleteMessages).not.toHaveBeenCalled();+ const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);+ expect(deleteUserById).not.toHaveBeenCalled();});it('should remove the user from all groups via $pullAll', async () => {@@ -393,6 +491,38 @@ describe('deleteUserController', () => {expect(mockRes.status).toHaveBeenCalledWith(500);expect(mockRes.json).toHaveBeenCalledWith({ message: 'Something went wrong.' });+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(+ userId.toString(),+ expect.any(Date),+ );+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(+ userId.toString(),+ expect.any(Date),+ );+ expect(deleteUserById).not.toHaveBeenCalled();+ });++ it('preserves queued trigger payloads when deletion fails before the user commit', async () => {+ const userId = new mongoose.Types.ObjectId();+ deleteMessages.mockRejectedValueOnce(new Error('db down'));++ await deleteUserController({ user: { id: String(userId), _id: userId } }, mockRes);++ expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));+ });++ it('does not purge trigger payloads when the user deletion did not commit', async () => {+ const userId = new mongoose.Types.ObjectId();+ deleteUserById.mockResolvedValueOnce({ deletedCount: 0 });++ await deleteUserController({ user: { id: String(userId), _id: userId } }, mockRes);++ expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));+ expect(mockRes.status).toHaveBeenCalledWith(500);});it('should use string user.id (not ObjectId user._id) for memberIds removal', async () => {diff --git a/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js b/api/server/controllers/__tests__/UserController.mcpOAuth.spec.jsindex 3a4b365646..545283ce26 100644--- a/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js+++ b/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js@@ -90,6 +90,13 @@ jest.mock('~/server/services/Files/process', () => ({processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),}));+jest.mock('~/server/services/Agents/triggers', () => ({+ drainAgentTriggerDeliveriesForUser: jest.fn(),+ prepareAgentTriggerUserPurge: jest.fn(),+ cancelAgentTriggerUserPurge: jest.fn(),+ purgeAgentTriggerDeliveriesForUser: jest.fn(),+}));+jest.mock('~/server/services/Config', () => ({getAppConfig: (...args) => mockGetAppConfig(...args),}));diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.jsindex 6198122bd0..9af915b772 100644--- a/api/server/controllers/__tests__/deleteUser.spec.js+++ b/api/server/controllers/__tests__/deleteUser.spec.js@@ -19,6 +19,14 @@ const mockDeleteToolCalls = jest.fn();const mockDeleteUserAgents = jest.fn();const mockDeleteUserPrompts = jest.fn();const mockDeleteUserSkills = jest.fn();+const mockGetCleanupBlockingJobIdsForUser = jest.fn();+const mockAbortJob = jest.fn();+const mockDrainAgentTriggerDeliveriesForUser = jest.fn();+const mockPrepareAgentTriggerUserPurge = jest.fn();+const mockCancelAgentTriggerUserPurge = jest.fn();+const mockPurgeAgentTriggerDeliveriesForUser = jest.fn();+const mockBeginAgentTriggerUserDeletion = jest.fn();+const mockCancelAgentTriggerUserDeletion = jest.fn();jest.mock('@librechat/data-schemas', () => ({logger: { error: jest.fn(), info: jest.fn() },@@ -40,6 +48,10 @@ jest.mock('@librechat/api', () => ({needsRefresh: jest.fn(),getNewS3URL: jest.fn(),deleteAllSharedLinksWithCleanup: (...args) => mockDeleteAllSharedLinksWithCleanup(...args),+ GenerationJobManager: {+ getCleanupBlockingJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args),+ abortJob: (...args) => mockAbortJob(...args),+ },}));jest.mock('~/models', () => ({@@ -47,6 +59,8 @@ jest.mock('~/models', () => ({deleteAllSharedLinks: (...args) => mockDeleteAllSharedLinks(...args),updateUserPlugins: (...args) => mockUpdateUserPlugins(...args),deleteUserById: (...args) => mockDeleteUserById(...args),+ beginAgentTriggerUserDeletion: (...args) => mockBeginAgentTriggerUserDeletion(...args),+ cancelAgentTriggerUserDeletion: (...args) => mockCancelAgentTriggerUserDeletion(...args),deleteMessages: (...args) => mockDeleteMessages(...args),deletePresets: (...args) => mockDeletePresets(...args),deleteUserKey: (...args) => mockDeleteUserKey(...args),@@ -101,6 +115,13 @@ jest.mock('~/server/services/Files/process', () => ({processDeleteRequest: (...args) => mockProcessDeleteRequest(...args),}));+jest.mock('~/server/services/Agents/triggers', () => ({+ drainAgentTriggerDeliveriesForUser: (...args) => mockDrainAgentTriggerDeliveriesForUser(...args),+ prepareAgentTriggerUserPurge: (...args) => mockPrepareAgentTriggerUserPurge(...args),+ cancelAgentTriggerUserPurge: (...args) => mockCancelAgentTriggerUserPurge(...args),+ purgeAgentTriggerDeliveriesForUser: (...args) => mockPurgeAgentTriggerDeliveriesForUser(...args),+}));+jest.mock('~/server/services/Config', () => ({getAppConfig: jest.fn(),}));@@ -126,7 +147,7 @@ function stubDeletionMocks() {mockDeletePresets.mockResolvedValue();mockDeleteConvos.mockResolvedValue();mockDeleteUserPluginAuth.mockResolvedValue();- mockDeleteUserById.mockResolvedValue();+ mockDeleteUserById.mockResolvedValue({ deletedCount: 1 });mockDeleteAllSharedLinks.mockResolvedValue();mockDeleteAllSharedLinksWithCleanup.mockResolvedValue({ deletedCount: 0 });mockGetFiles.mockResolvedValue([]);@@ -136,6 +157,14 @@ function stubDeletionMocks() {mockDeleteUserAgents.mockResolvedValue();mockDeleteUserPrompts.mockResolvedValue();mockDeleteUserSkills.mockResolvedValue(0);+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValue([]);+ mockAbortJob.mockResolvedValue({ success: true });+ mockDrainAgentTriggerDeliveriesForUser.mockResolvedValue();+ mockPrepareAgentTriggerUserPurge.mockResolvedValue();+ mockCancelAgentTriggerUserPurge.mockResolvedValue(true);+ mockPurgeAgentTriggerDeliveriesForUser.mockResolvedValue();+ mockBeginAgentTriggerUserDeletion.mockResolvedValue('acquired');+ mockCancelAgentTriggerUserDeletion.mockResolvedValue(true);}beforeEach(() => {@@ -158,12 +187,45 @@ describe('deleteUserController - 2FA enforcement', () => {expect(mockDeleteUserPrompts).toHaveBeenCalledWith('user1');expect(mockDeleteUserSkills).toHaveBeenCalledWith('user1');expect(mockVerifyOTPOrBackupCode).not.toHaveBeenCalled();+ expect(mockBeginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(+ mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0],+ );+ expect(mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0]).toBeLessThan(+ mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0],+ );+ expect(mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0]).toBeLessThan(+ mockDeleteMessages.mock.invocationCallOrder[0],+ );+ expect(mockDeleteMessages.mock.invocationCallOrder[0]).toBeLessThan(+ mockDeleteUserById.mock.invocationCallOrder[0],+ );+ expect(mockCancelAgentTriggerUserDeletion).not.toHaveBeenCalled();+ expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();+ });++ it('aborts active generation jobs before deleting account-owned records', async () => {+ const req = {+ user: { id: 'user1', _id: 'user1', email: '[email protected]', tenantId: 'tenant-1' },+ body: {},+ };+ const res = createRes();+ mockGetUserById.mockResolvedValue({ _id: 'user1', twoFactorEnabled: false });+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);++ await deleteUserController(req, res);++ expect(mockGetCleanupBlockingJobIdsForUser).toHaveBeenCalledWith('user1', 'tenant-1');+ expect(mockAbortJob).toHaveBeenCalledWith('stream-1', { awaitProviderDrain: true });+ expect(mockAbortJob.mock.invocationCallOrder[0]).toBeLessThan(+ mockDeleteMessages.mock.invocationCallOrder[0],+ );});it('proceeds with deletion when user has no 2FA record', async () => {const req = { user: { id: 'user1', _id: 'user1', email: '[email protected]' }, body: {} };const res = createRes();mockGetUserById.mockResolvedValue(null);+ mockBeginAgentTriggerUserDeletion.mockResolvedValueOnce('missing');await deleteUserController(req, res);diff --git a/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js b/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.jsindex fb46382ff6..30d687bf64 100644--- a/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js+++ b/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js@@ -101,6 +101,13 @@ jest.mock('~/server/services/Files/process', () => ({processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),}));+jest.mock('~/server/services/Agents/triggers', () => ({+ drainAgentTriggerDeliveriesForUser: jest.fn(),+ prepareAgentTriggerUserPurge: jest.fn(),+ cancelAgentTriggerUserPurge: jest.fn(),+ purgeAgentTriggerDeliveriesForUser: jest.fn(),+}));+jest.mock('~/server/services/Config', () => ({getAppConfig: jest.fn(),}));diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.jsindex bbdd026152..c2f1173257 100644--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js@@ -17,6 +17,7 @@ const mockTenantStorageRun = jest.fn(async (context, callback) => {const mockSaveMessage = jest.fn();const mockGetConvo = jest.fn();const mockGetMessages = jest.fn();+const mockIsAgentTriggerPrincipalActive = jest.fn();const mockFilterPersistableAbortContent = jest.fn((content) => content);const mockCheckAndIncrementPendingRequest = jest.fn();const mockDecrementPendingRequest = jest.fn();@@ -24,6 +25,8 @@ const mockGenerationJobManager = {createJob: jest.fn(),emitError: jest.fn(),completeJob: jest.fn(),+ beginProviderExecution: jest.fn(),+ markProviderExecutionDrained: jest.fn(),getResumeState: jest.fn(),updateMetadata: jest.fn(),claimGeneration: jest.fn(),@@ -100,6 +103,7 @@ jest.mock('~/models', () => ({saveMessage: (...args) => mockSaveMessage(...args),getMessages: (...args) => mockGetMessages(...args),getConvo: (...args) => mockGetConvo(...args),+ isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),}));const AgentController = require('../request');@@ -112,9 +116,12 @@ describe('ResumableAgentController tenant context', () => {mockDecrementPendingRequest.mockResolvedValue(undefined);mockGetConvo.mockResolvedValue({ createdAt: '2026-07-31T00:00:00.000Z' });mockGetMessages.mockResolvedValue([]);+ mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);mockGenerationJobManager.emitError.mockResolvedValue(undefined);mockGenerationJobManager.completeJob.mockResolvedValue(undefined);+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);+ mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);mockGenerationJobManager.hasJob.mockResolvedValue(true);@@ -130,6 +137,10 @@ describe('ResumableAgentController tenant context', () => {let allSubscribersLeftHandler;mockGenerationJobManager.createJob.mockResolvedValue({createdAt: 1000,+ metadata: {+ providerExecutionId: 'provider-segment-1',+ providerDrained: true,+ },readyPromise: Promise.resolve(),abortController: new AbortController(),emitter: {diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex ceb1abb3ac..2ddc02abac 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -17,6 +17,8 @@ const mockGenerationJobManager = {publishTerminalClaim: jest.fn(),finishTerminalJob: jest.fn(),completeJob: jest.fn(),+ beginProviderExecution: jest.fn(),+ markProviderExecutionDrained: jest.fn(),failPausePersistence: jest.fn(),getResumeState: jest.fn(),updateMetadata: jest.fn(),@@ -61,6 +63,7 @@ const mockFilterPersistableAbortContent = jest.fn((content) =>const mockGetConvo = jest.fn();const mockGetMessages = jest.fn();const mockSaveMessage = jest.fn();+const mockIsAgentTriggerPrincipalActive = jest.fn();const mockDeleteAgentCheckpoint = jest.fn();const mockStartupTelemetry = {mark: jest.fn(),@@ -208,6 +211,7 @@ jest.mock('~/models', () => ({saveMessage: (...args) => mockSaveMessage(...args),getMessages: (...args) => mockGetMessages(...args),getConvo: (...args) => mockGetConvo(...args),+ isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),}));const AgentController = require('../request');@@ -245,9 +249,14 @@ describe('ResumableAgentController resume metadata', () => {mockDecrementPendingRequest.mockResolvedValue(undefined);mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' });mockGetMessages.mockResolvedValue([]);+ mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);mockGenerationJobManager.createJob.mockResolvedValue({createdAt: 1000,- metadata: { checkpointNamespace: '1000' },+ metadata: {+ checkpointNamespace: '1000',+ providerExecutionId: 'provider-segment-1',+ providerDrained: true,+ },readyPromise: Promise.resolve(),abortController: new AbortController(),emitter: { on: jest.fn() },@@ -278,6 +287,8 @@ describe('ResumableAgentController resume metadata', () => {);mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);mockGenerationJobManager.completeJob.mockResolvedValue(undefined);+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);+ mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());mockGenerationJobManager.resumeClaimedGeneration.mockResolvedValue(null);@@ -621,6 +632,91 @@ describe('ResumableAgentController resume metadata', () => {});});+ it('rejects any session fenced after authentication before generation execution starts', async () => {+ mockIsAgentTriggerPrincipalActive.mockResolvedValue(false);+ const req = {+ user: { id: 'user-123' },+ body: {+ text: 'Run after a slow trigger admission.',+ messageId: 'user-message',+ clientRequestId: 'trigger-request',+ conversationId: 'conversation-123',+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },+ },+ config: {},+ };+ const res = createResumableResponse();+ const initializeClient = jest.fn();++ await AgentController(req, res, jest.fn(), initializeClient, null);++ expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1);+ expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');+ expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(+ mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0],+ );+ expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();+ expect(res.status).toHaveBeenCalledWith(409);+ expect(res.json).toHaveBeenCalledWith({+ status: 409,+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ error: 'Account deletion is in progress',+ generationProtocolVersion: 1,+ });+ expect(initializeClient).not.toHaveBeenCalled();+ expect(mockAcceptAgentStartupTelemetry).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(+ 'conversation-123',+ JSON.stringify({+ status: 409,+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ error: 'Account deletion is in progress',+ }),+ 1000,+ );+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(+ 'user-123',+ 'trigger-request',+ 'conversation-123',+ DEFAULT_OWNED_CLAIM,+ );+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');+ });++ it('does not start a provider when account deletion or replacement wins the startup CAS', async () => {+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(false);+ const req = {+ user: { id: 'user-123' },+ body: {+ text: 'Race destructive cleanup.',+ messageId: 'user-message',+ conversationId: 'conversation-123',+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },+ },+ config: {},+ };+ const res = createResumableResponse();+ const initializeClient = jest.fn();++ await AgentController(req, res, jest.fn(), initializeClient, null);++ expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');+ expect(mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0]).toBeLessThan(+ mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],+ );+ expect(initializeClient).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(+ 'conversation-123',+ expect.stringContaining('Generation stopped before provider startup'),+ 1000,+ );+ expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(+ 'conversation-123',+ 1000,+ 'provider-segment-1',+ );+ });+it('prefetches conversation state before admission and joins it with job metadata', async () => {let resolveConversation;let signalMetadataStarted;@@ -3208,7 +3304,7 @@ describe('ResumableAgentController resume metadata', () => {getHaltReason: () => 'preempt_incomplete',};- const runFirstTurn = async ({ run } = {}) => {+ const runFirstTurn = async ({ run, addTitle: suppliedAddTitle } = {}) => {let signalFinished;const finished = new Promise((resolve) => {signalFinished = resolve;@@ -3216,9 +3312,11 @@ describe('ResumableAgentController resume metadata', () => {mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());let titleSignal;- const addTitle = jest.fn(async (_req, options) => {- titleSignal = options?.signal;- });+ const addTitle =+ suppliedAddTitle ??+ jest.fn(async (_req, options) => {+ titleSignal = options?.signal;+ });const client = {options: {},@@ -3290,6 +3388,30 @@ describe('ResumableAgentController resume metadata', () => {);});+ it('does not acknowledge provider drain until deferred title persistence settles', async () => {+ resolveTitleTiming.mockReturnValueOnce('final');+ let resolveTitle;+ const titlePending = new Promise((resolve) => {+ resolveTitle = resolve;+ });+ const addTitle = jest.fn(() => titlePending);++ await runFirstTurn({ addTitle });++ expect(addTitle).toHaveBeenCalledTimes(1);+ expect(mockGenerationJobManager.markProviderExecutionDrained).not.toHaveBeenCalled();++ resolveTitle();+ await nextTick();+ await nextTick();++ expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(+ expect.any(String),+ 1000,+ 'provider-segment-1',+ );+ });+it('cancels an in-flight immediate title when the turn ends preempt-incomplete', async () => {const { addTitle, getTitleSignal } = await runFirstTurn({ run: preemptIncompleteRun });diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.jsindex 0cfb9f13d1..eb30808054 100644--- a/api/server/controllers/agents/__tests__/resume.spec.js+++ b/api/server/controllers/agents/__tests__/resume.spec.js@@ -60,6 +60,8 @@ const mockGenerationJobManager = {publishTerminalClaim: jest.fn(),finishTerminalJob: jest.fn(),completeJob: jest.fn(),+ beginProviderExecution: jest.fn(),+ markProviderExecutionDrained: jest.fn(),failPausePersistence: jest.fn(),expireApproval: jest.fn(),approvals: {@@ -246,6 +248,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {);mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);mockGenerationJobManager.completeJob.mockResolvedValue(true);+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);+ mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);@@ -365,6 +369,32 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {// ...then torn down exactly once in the finally.expect(mockCleanupMCPRequestContextForReq).toHaveBeenCalledTimes(1);});++ it('marks the exact resumed provider segment drained only after request cleanup', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());+ const res = await post(approveBody());+ expect(res.status).toBe(200);+ await settled;+ await flush();++ const resumePatch = mockGenerationJobManager.approvals.resolve.mock.calls[0][2];+ expect(mockGenerationJobManager.beginProviderExecution).toHaveBeenCalledWith(+ CONVO_ID,+ 1000,+ resumePatch.providerExecutionId,+ );+ expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(+ CONVO_ID,+ 1000,+ resumePatch.providerExecutionId,+ );+ expect(mockCleanupMCPRequestContextForReq.mock.invocationCallOrder[0]).toBeLessThan(+ mockGenerationJobManager.markProviderExecutionDrained.mock.invocationCallOrder[0],+ );+ expect(+ mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],+ ).toBeLessThan(mockInitializeClient.mock.invocationCallOrder[0]);+ });});describe('request guards (rejected before claiming the action)', () => {@@ -594,7 +624,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- { preemptCapable: true },+ expect.objectContaining({+ preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,+ }),1000,);await settled;@@ -687,7 +721,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- { preemptCapable: true },+ expect.objectContaining({+ preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,+ }),1000,);await settled;@@ -726,7 +764,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- { preemptCapable: true },+ expect.objectContaining({+ preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,+ }),1000,);expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);@@ -747,7 +789,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- { preemptCapable: true },+ expect.objectContaining({+ preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,+ }),1000,);expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);@@ -784,7 +830,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- { preemptCapable: true },+ expect.objectContaining({+ preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,+ }),1000,);expect(mockCaptureAgentCheckpointGeneration.mock.invocationCallOrder[0]).toBeLessThan(@@ -924,7 +974,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- { preemptCapable: true },+ expect.objectContaining({+ preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,+ }),1000,);});@@ -1408,8 +1462,10 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- {+ expect.objectContaining({preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,resolvedAskUserQuestions: [{request: 'What should I name the file?',@@ -1417,7 +1473,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {contentIndex: 1,},],- },+ }),1000,);expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(@@ -1449,8 +1505,10 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- {+ expect.objectContaining({preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,resolvedAskUserQuestions: [{request: 'What should I name the file?',@@ -1458,7 +1516,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {contentMissing: true,},],- },+ }),1000,);});@@ -1484,8 +1542,10 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID,ACTION_ID,- {+ expect.objectContaining({preemptCapable: true,+ providerExecutionId: expect.any(String),+ providerDrained: true,resolvedAskUserQuestions: [{request: { questions: expect.any(Array) },@@ -1493,7 +1553,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {toolCallId: 'tc1',},],- },+ }),1000,);});diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex ce1bdf5db9..60f3bcf9de 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -33,7 +33,7 @@ const {cleanupMCPRequestContextForReq,} = require('~/server/services/MCPRequestContext');const { logViolation } = require('~/cache');-const { saveMessage, getMessages, getConvo } = require('~/models');+const { saveMessage, getMessages, getConvo, isAgentTriggerPrincipalActive } = require('~/models');const {GENERATION_PROTOCOL_HEADER,GENERATION_PROTOCOL_V2,@@ -858,6 +858,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitlet client = null;let jobCreatedAt;+ let providerExecutionId;try {logger.debug(`[ResumableAgentController] Creating job`, {@@ -901,10 +902,38 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit},});startupTelemetry?.mark('job_created');- acceptAgentStartupTelemetry(req, streamId);- startupTelemetry?.mark('metadata_persisted');generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);jobCreatedAt = job.createdAt; // Capture creation time to detect job replacement+ providerExecutionId = job.metadata?.providerExecutionId;++ /** Authentication can precede a slow admission path. Recheck the durable+ * account-deletion fence after the job is committed but before execution+ * starts. This ordering closes both sides of the race for ordinary and+ * trigger-scoped sessions: a fence that wins first rejects this run; a+ * fence that starts after this read must observe the already-created job+ * in account deletion's active-generation drain. */+ if (!(await isAgentTriggerPrincipalActive(userId))) {+ throw Object.assign(new Error('Account deletion is in progress'), {+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ status: 409,+ });+ }+ if (+ providerExecutionId &&+ !(await GenerationJobManager.beginProviderExecution(+ streamId,+ jobCreatedAt,+ providerExecutionId,+ ))+ ) {+ throw Object.assign(new Error('Generation stopped before provider startup'), {+ code: 'RUN_REPLACED',+ status: 409,+ });+ }++ acceptAgentStartupTelemetry(req, streamId);+ startupTelemetry?.mark('metadata_persisted');req._resumableStreamId = streamId;getMCPRequestContext(req, undefined, { cleanupOnResponse: false });let recoveredSteerCommitted = false;@@ -1068,6 +1097,18 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitdisposeClient(client);}client = null;+ if (providerExecutionId) {+ await GenerationJobManager.markProviderExecutionDrained?.(+ streamId,+ jobCreatedAt,+ providerExecutionId,+ ).catch((drainError) => {+ logger.warn(+ '[ResumableAgentController] Failed to record initialization-abort provider drain',+ drainError,+ );+ });+ }}return;}@@ -1109,6 +1150,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit};let immediateTitlePromise = null;+ let trailingWritePromise = null;let backgroundClientCleanupScheduled = false;let terminalClaim = null;let terminalClaimFinished = false;@@ -1720,7 +1762,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitdisposeClient(client);}} else if (shouldGenerateTitle) {- addTitle(req, {+ trailingWritePromise = addTitle(req, {text: text || getAttachmentTitleText(req.body.files),response: { ...response },client,@@ -1816,27 +1858,44 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit};// Start generation and handle any unhandled errors- startGeneration().catch(async (err) => {- logger.error(- `[ResumableAgentController] Unhandled error in background generation: ${err.message}`,- );- startupTelemetry?.end('error', err);- if (!pausePersistenceFailed) {- await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(- (completeErr) => {- logger.warn(- '[ResumableAgentController] completeJob failed during background-error cleanup',- completeErr,- );- },+ void startGeneration()+ .catch(async (err) => {+ logger.error(+ `[ResumableAgentController] Unhandled error in background generation: ${err.message}`,);- }- try {- await finishResumableRequest(req, userId);- } finally {- disposeBackgroundClient();- }- });+ startupTelemetry?.end('error', err);+ if (!pausePersistenceFailed) {+ await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(+ (completeErr) => {+ logger.warn(+ '[ResumableAgentController] completeJob failed during background-error cleanup',+ completeErr,+ );+ },+ );+ }+ try {+ await finishResumableRequest(req, userId);+ } finally {+ disposeBackgroundClient();+ }+ })+ .finally(async () => {+ await Promise.allSettled([immediateTitlePromise, trailingWritePromise].filter(Boolean));+ if (providerExecutionId) {+ await GenerationJobManager.markProviderExecutionDrained?.(+ streamId,+ jobCreatedAt,+ providerExecutionId,+ );+ }+ })+ .catch((drainError) => {+ logger.warn(+ '[ResumableAgentController] Failed to record completed provider drain',+ drainError,+ );+ });} catch (error) {logger.error('[ResumableAgentController] Initialization error:', error);const initializationFailure = getInitializationFailure(error);@@ -1939,6 +1998,18 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitif (client) {disposeClient(client);}+ if (jobCreatedAt != null && providerExecutionId) {+ await GenerationJobManager.markProviderExecutionDrained?.(+ streamId,+ jobCreatedAt,+ providerExecutionId,+ ).catch((drainError) => {+ logger.warn(+ '[ResumableAgentController] Failed to record initialization-error provider drain',+ drainError,+ );+ });+ }}};diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.jsindex a5bae98f76..cb3ea82908 100644--- a/api/server/controllers/agents/resume.js+++ b/api/server/controllers/agents/resume.js@@ -1,3 +1,4 @@+const { randomUUID } = require('crypto');const { logger } = require('@librechat/data-schemas');const { Constants, EModelEndpoint } = require('librechat-data-provider');const {@@ -729,6 +730,7 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)// the user when they retry the still-paused approval. Release the slot on that path too.let claimed;let checkpointGeneration;+ const providerExecutionId = randomUUID();try {checkpointGeneration = await checkpointGenerationPromise;/** The CAS that reopens steering must also publish THIS owner's seal@@ -739,6 +741,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)pendingAction.actionId,{preemptCapable: isSteerPreemptSupported(),+ providerExecutionId,+ providerDrained: true,...(resolvedAskUserQuestion && { resolvedAskUserQuestions }),},job.createdAt,@@ -902,6 +906,17 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)let pausePersistenceFailed = false;let pausePersistenceFailureFinalized = false;try {+ if (+ !(await GenerationJobManager.beginProviderExecution(+ streamId,+ job.createdAt,+ providerExecutionId,+ ))+ ) {+ throw Object.assign(new Error('Generation stopped before provider resume'), {+ code: 'RUN_REPLACED',+ });+ }const result = await initializeClient({req,res,@@ -1091,17 +1106,27 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)}}} finally {- // Tear down the MCP request-context store seeded before the ACK (parity with- // request.js's finishResumableRequest). No-op if it was never seeded.- await cleanupMCPRequestContextForReq(req);- // Release the concurrency slot taken above — UNLESS handleRunInterrupt already- // released it on a re-pause (so a fast /resume isn't 429'd). On a normal finish or- // error it didn't, so release here. A re-pause re-acquires its own slot next resume.- if (!client?.pendingRequestReleased) {- await decrementPendingRequest(userId);- }- if (client) {- disposeClient(client);+ try {+ // Tear down the MCP request-context store seeded before the ACK (parity with+ // request.js's finishResumableRequest). No-op if it was never seeded.+ await cleanupMCPRequestContextForReq(req);+ // Release the concurrency slot taken above — UNLESS handleRunInterrupt already+ // released it on a re-pause (so a fast /resume isn't 429'd). On a normal finish or+ // error it didn't, so release here. A re-pause re-acquires its own slot next resume.+ if (!client?.pendingRequestReleased) {+ await decrementPendingRequest(userId);+ }+ if (client) {+ disposeClient(client);+ }+ } finally {+ await GenerationJobManager.markProviderExecutionDrained?.(+ streamId,+ job.createdAt,+ providerExecutionId,+ ).catch((drainError) => {+ logger.warn('[ResumeAgentController] Failed to record provider drain', drainError);+ });}}};diff --git a/api/server/experimental.js b/api/server/experimental.jsindex a259a8e31c..387e6940a5 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -25,6 +25,7 @@ const {preAuthTenantMiddleware,requestContextMiddleware,configureServerTimeouts,+ setupGracefulShutdown,configureMessageFilterRegexValidator,configureFileConfigRegexEngine,} = require('@librechat/api');@@ -34,6 +35,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilitiesconst createValidateImageRequest = require('./middleware/validateImageRequest');const { startExpiredFileSweep } = require('./services/Files/process');const { initializeGitHubSkillSync } = require('./services/Skills/sync');+const { initializeAgentTriggerService } = require('./services/Agents/triggers');const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');const { updateInterfacePermissions: updateInterfacePerms } = require('@librechat/api');const {@@ -159,6 +161,8 @@ if (cluster.isMaster) {const listeningWorkers = new Set();let retentionSweepWorkerId = null;const startTime = Date.now();+ let shuttingDown = false;+ let remainingShutdownWorkers = 0;const assignRetentionSweepWorker = () => {if (retentionSweepWorkerId && cluster.workers[retentionSweepWorkerId]) {@@ -229,15 +233,32 @@ if (cluster.isMaster) {logger.error(`Worker ${worker.process.pid} died (${activeWorkers}/${workers}). Code: ${code}, Signal: ${signal}`,);+ if (shuttingDown) {+ remainingShutdownWorkers = Math.max(0, remainingShutdownWorkers - 1);+ if (remainingShutdownWorkers === 0) {+ process.exit(0);+ }+ return;+ }logger.info('Starting a new worker to replace it...');cluster.fork();});/** Graceful shutdown on SIGTERM/SIGINT */const shutdown = () => {+ if (shuttingDown) {+ return;+ }+ shuttingDown = true;logger.info('Master received shutdown signal, terminating workers...');- for (const id in cluster.workers) {- cluster.workers[id].kill();+ const liveWorkers = Object.values(cluster.workers).filter(Boolean);+ remainingShutdownWorkers = liveWorkers.length;+ if (remainingShutdownWorkers === 0) {+ process.exit(0);+ return;+ }+ for (const worker of liveWorkers) {+ worker.kill();}setTimeout(() => {logger.info('Forcing shutdown after timeout');@@ -499,6 +520,7 @@ if (cluster.isMaster) {await initializeMCPs();await initializeOAuthReconnectManager();await checkMigrations();+ await initializeAgentTriggerService({ address: server.address() });} catch (initErr) {logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);process.exit(1);@@ -512,6 +534,7 @@ if (cluster.isMaster) {headersTimeout: server.headersTimeout,requestTimeout: server.requestTimeout,});+ setupGracefulShutdown(server);};startServer().catch((err) => {diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.jsindex 2287011154..c4364fb6b8 100644--- a/api/server/experimental.spec.js+++ b/api/server/experimental.spec.js@@ -13,6 +13,16 @@ describe('Experimental server configuration', () => {expect(listenIndex).toBeLessThan(timeoutConfigIndex);});+ it('lets each worker drain registered services before cluster shutdown', () => {+ const listenIndex = source.indexOf('const server = app.listen');+ const gracefulShutdownIndex = source.indexOf('setupGracefulShutdown(server);');++ expect(gracefulShutdownIndex).toBeGreaterThan(-1);+ expect(listenIndex).toBeLessThan(gracefulShutdownIndex);+ expect(source).toContain('if (shuttingDown) {');+ expect(source).toMatch(/if \(shuttingDown\) \{[\s\S]*?return;[\s\S]*?Starting a new worker/);+ });+it('runs cross-tenant startup work in the system context', () => {expect(source).toContain('await runAsSystem(seedDatabase);');expect(source).toMatch(diff --git a/api/server/index.js b/api/server/index.jsindex 6732f1eaf3..87aed78629 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -56,6 +56,7 @@ const initializeOAuthReconnectManager = require('./services/initializeOAuthReconconst { capabilityContextMiddleware } = require('./middleware/roles/capabilities');const createValidateImageRequest = require('./middleware/validateImageRequest');const { initializeGitHubSkillSync } = require('./services/Skills/sync');+const { initializeAgentTriggerService } = require('./services/Agents/triggers');const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');const { startExpiredFileSweep } = require('./services/Files/process');const { checkMigrations } = require('./services/start/migration');@@ -397,6 +398,7 @@ const startServer = async () => {if (inspectFlags || isEnabled(process.env.MEM_DIAG)) {memoryDiagnostics.start();}+ await initializeAgentTriggerService({ address: server.address() });serverReady = true;logger.info('Server readiness checks passing.');} catch (initErr) {diff --git a/api/server/index.metrics.spec.js b/api/server/index.metrics.spec.jsindex b611488cd9..6b95104c3d 100644--- a/api/server/index.metrics.spec.js+++ b/api/server/index.metrics.spec.js@@ -34,6 +34,10 @@ jest.mock('~/config', () => ({}),}));+jest.mock('~/server/services/Agents/triggers', () => ({+ initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),+}));+describe('Server metrics route', () => {jest.setTimeout(30_000);diff --git a/api/server/index.spec.js b/api/server/index.spec.jsindex bdc249bcc7..6461a148e3 100644--- a/api/server/index.spec.js+++ b/api/server/index.spec.js@@ -35,6 +35,10 @@ jest.mock('~/config', () => ({}),}));+jest.mock('~/server/services/Agents/triggers', () => ({+ initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),+}));+jest.mock('@librechat/api/telemetry',() => ({@@ -164,6 +168,14 @@ describe('Startup readiness wiring', () => {expect(agentsRouteIndex).toBeGreaterThan(-1);expect(readinessGateIndex).toBeLessThan(agentsRouteIndex);});++ it('awaits durable trigger delivery before reporting readiness', () => {+ const triggerDeliveryIndex = source.indexOf('await initializeAgentTriggerService(');+ const readyIndex = source.indexOf('serverReady = true;');++ expect(triggerDeliveryIndex).toBeGreaterThan(-1);+ expect(readyIndex).toBeGreaterThan(triggerDeliveryIndex);+ });});describe('Server Configuration', () => {diff --git a/api/server/middleware/__tests__/requireJwtAuth.spec.js b/api/server/middleware/__tests__/requireJwtAuth.spec.jsindex 08f779fac0..bbadd5fb04 100644--- a/api/server/middleware/__tests__/requireJwtAuth.spec.js+++ b/api/server/middleware/__tests__/requireJwtAuth.spec.js@@ -253,6 +253,30 @@ describe('requireJwtAuth tenant context chaining', () => {expect(logger.warn).not.toHaveBeenCalled();});+ it('preserves the typed account-deletion fence in an authentication rejection', () => {+ const req = mockReq(undefined, {+ _mockStrategies: {+ jwt: {+ user: false,+ info: {+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ },+ status: 401,+ },+ },+ });+ const res = mockRes();++ requireJwtAuth(req, res, jest.fn());++ expect(res.status).toHaveBeenCalledWith(401);+ expect(res.json).toHaveBeenCalledWith({+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ });+ });+it('logs OpenID JWT expiry when JWT fallback succeeds', () => {isEnabled.mockReturnValue(true);mockRegisteredStrategies.add('openidJwt');diff --git a/api/server/middleware/requireJwtAuth.js b/api/server/middleware/requireJwtAuth.jsindex 251fef9958..74232012f2 100644--- a/api/server/middleware/requireJwtAuth.js+++ b/api/server/middleware/requireJwtAuth.js@@ -17,6 +17,7 @@ const hasPassportStrategy = (strategy) =>const getAuthenticatedUserId = (user) => user?.id?.toString?.() ?? user?._id?.toString?.();const refreshCloudFrontCookies =maybeRefreshCloudFrontAuthCookiesMiddleware ?? ((_req, _res, next) => next());+const ACCOUNT_DELETION_CODE = 'ACCOUNT_DELETION_IN_PROGRESS';const getAuthTokenSource = (req) => {const authorization = req.headers.authorization;@@ -166,6 +167,7 @@ const requireJwtAuth = (req, res, next) => {logAuthenticationFailure({ strategy, info, status, err });return res.status(status || 401).json({message: info?.message || 'Unauthorized',+ ...(info?.code === ACCOUNT_DELETION_CODE && { code: ACCOUNT_DELETION_CODE }),});}if (strategy === 'openidJwt' && getAuthenticatedUserId(user) !== openIdReuseUserId) {diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.jsindex 20d4eb1797..0aa157c179 100644--- a/api/server/routes/admin/users.js+++ b/api/server/routes/admin/users.js@@ -3,6 +3,12 @@ const { createAdminUsersHandlers } = require('@librechat/api');const { SystemCapabilities } = require('@librechat/data-schemas');const { requireCapability } = require('~/server/middleware/roles/capabilities');const { requireJwtAuth } = require('~/server/middleware');+const {+ drainAgentTriggerDeliveriesForUser,+ prepareAgentTriggerUserPurge,+ cancelAgentTriggerUserPurge,+ purgeAgentTriggerDeliveriesForUser,+} = require('~/server/services/Agents/triggers');const db = require('~/models');const router = express.Router();@@ -14,6 +20,12 @@ const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS);const handlers = createAdminUsersHandlers({findUsers: db.findUsers,countUsers: db.countUsers,+ beginAgentTriggerUserDeletion: db.beginAgentTriggerUserDeletion,+ cancelAgentTriggerUserDeletion: db.cancelAgentTriggerUserDeletion,+ drainAgentTriggerDeliveriesForUser,+ prepareAgentTriggerUserPurge,+ cancelAgentTriggerUserPurge,+ purgeAgentTriggerDeliveriesForUser,deleteUserById: db.deleteUserById,deleteConfig: db.deleteConfig,deleteAclEntries: db.deleteAclEntries,diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.jsindex 987fba06df..ee3ed4e7bb 100644--- a/api/server/routes/agents/index.js+++ b/api/server/routes/agents/index.js@@ -13,6 +13,8 @@ const {attachAskUserQuestionAnswers,attachAskUserQuestionArgs,createMessageFilterPii,+ isAgentTriggerRequest,+ exemptAgentTriggerFromIpLimiter,} = require('@librechat/api');const { createSseStreamTelemetry } = require('@librechat/api/telemetry');const { logger } = require('@librechat/data-schemas');@@ -41,6 +43,10 @@ const chat = require('./chat');const { LIMIT_MESSAGE_IP, LIMIT_MESSAGE_USER } = process.env ?? {};+/** Applies `limiter` unless this trusted loopback request should skip it. */+const unless = (isExempt, limiter) => (req, res, next) =>+ isExempt(req) ? next() : limiter(req, res, next);+/** Untenanted jobs (pre-multi-tenancy) remain accessible if the userId check passes. */function hasTenantMismatch(job, user) {return job.metadata?.tenantId != null && job.metadata.tenantId !== user.tenantId;@@ -109,6 +115,12 @@ router.use('/v1/responses', responses);router.use('/v1', openai);router.use(requireJwtAuth);+// Capture the short-lived trigger identity immediately after authentication. Downstream+// middleware reads this stable decision instead of re-verifying an expired token.+router.use((req, _res, next) => {+ req._isAgentTrigger = isAgentTriggerRequest(req);+ next();+});router.use(checkBan);router.use(uaParser);@@ -899,7 +911,7 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {*/const steerLimiters = [];if (isEnabled(LIMIT_MESSAGE_IP)) {- steerLimiters.push(messageIpLimiter);+ steerLimiters.push(unless(exemptAgentTriggerFromIpLimiter, messageIpLimiter));}if (isEnabled(LIMIT_MESSAGE_USER)) {steerLimiters.push(messageUserLimiter);@@ -961,7 +973,7 @@ const chatRouter = express.Router();chatRouter.use(configMiddleware);if (isEnabled(LIMIT_MESSAGE_IP)) {- chatRouter.use(messageIpLimiter);+ chatRouter.use(unless(exemptAgentTriggerFromIpLimiter, messageIpLimiter));}if (isEnabled(LIMIT_MESSAGE_USER)) {diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.jsnew file mode 100644index 0000000000..7beb92f8ee--- /dev/null+++ b/api/server/services/Agents/triggers.js@@ -0,0 +1,21 @@+const { createAgentTriggerService } = require('@librechat/api');+const methods = require('~/models');++const service = createAgentTriggerService({+ methods,+ isPrincipalActive: methods.isAgentTriggerPrincipalActive,+});++module.exports = {+ initializeAgentTriggerService: service.initialize,+ stopAgentTriggerService: service.stop,+ dispatchAgentTrigger: service.dispatch,+ enqueueAgentTrigger: service.enqueue,+ getAgentTriggerDelivery: service.getDelivery,+ getAgentTriggerDeadLetters: service.getDeadLetters,+ requeueAgentTrigger: service.requeue,+ drainAgentTriggerDeliveriesForUser: service.drainUser,+ prepareAgentTriggerUserPurge: service.prepareUserPurge,+ cancelAgentTriggerUserPurge: service.cancelUserPurge,+ purgeAgentTriggerDeliveriesForUser: service.purgeUser,+};diff --git a/api/strategies/jwtStrategy.js b/api/strategies/jwtStrategy.jsindex 7a41f553e2..d5de52276b 100644--- a/api/strategies/jwtStrategy.js+++ b/api/strategies/jwtStrategy.js@@ -1,25 +1,59 @@-const { logger } = require('@librechat/data-schemas');+const { AGENT_TRIGGER_SCOPE } = require('@librechat/api');+const { logger, runAsSystem } = require('@librechat/data-schemas');const { SystemRoles } = require('librechat-data-provider');const { Strategy: JwtStrategy, ExtractJwt } = require('passport-jwt');const { getUserById, updateUser } = require('~/models');+const AGENT_TRIGGER_ADMISSION_PATHS = ['/api/agents/chat/agents', '/api/agents/chat/steer/deliver'];++/** A trigger-scoped bearer is deliberately useless outside the two internal+ * admission routes that the execution host calls. */+function isAgentTriggerAdmissionRequest(req) {+ if (req?.method !== 'POST' || req?.headers?.['x-lc-agent-trigger'] !== '1') {+ return false;+ }+ try {+ const pathname = new URL(req.originalUrl ?? req.url ?? '', 'http://localhost').pathname;+ return AGENT_TRIGGER_ADMISSION_PATHS.some((path) => pathname.endsWith(path));+ } catch {+ return false;+ }+}+// JWT strategyconst jwtLogin = () =>new JwtStrategy({jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),secretOrKey: process.env.JWT_SECRET,+ passReqToCallback: true,},- async (payload, done) => {+ async (req, payload, done) => {try {- const user = await getUserById(payload?.id, '-password -__v -totpSecret -backupCodes');+ if (payload?.scope === AGENT_TRIGGER_SCOPE && !isAgentTriggerAdmissionRequest(req)) {+ done(null, false, { message: 'Agent trigger token is not valid for this endpoint' });+ return;+ }+ const user = await runAsSystem(() =>+ getUserById(+ payload?.id,+ '-password -__v -totpSecret -backupCodes +agentTriggerDeletionStartedAt',+ ),+ );+ if (user?.agentTriggerDeletionStartedAt != null) {+ done(null, false, {+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ });+ return;+ }if (user) {user.id = user._id.toString();/** Absent on the full doc means local user; null skips getUserPrincipals' fallback lookup */user.idOnTheSource ??= null;if (!user.role) {user.role = SystemRoles.USER;- await updateUser(user.id, { role: user.role });+ await runAsSystem(() => updateUser(user.id, { role: user.role }));}done(null, user);} else {diff --git a/api/strategies/jwtStrategy.spec.js b/api/strategies/jwtStrategy.spec.jsindex 3a8b83812d..da405ecb0c 100644--- a/api/strategies/jwtStrategy.spec.js+++ b/api/strategies/jwtStrategy.spec.js@@ -1,8 +1,11 @@const { SystemRoles } = require('librechat-data-provider');let capturedVerifyCallback;+let capturedStrategyOptions;+const mockRunAsSystem = jest.fn((callback) => callback());jest.mock('passport-jwt', () => ({Strategy: jest.fn((opts, verifyCallback) => {+ capturedStrategyOptions = opts;capturedVerifyCallback = verifyCallback;return { name: 'jwt' };}),@@ -13,6 +16,11 @@ jest.mock('passport-jwt', () => ({jest.mock('@librechat/data-schemas', () => ({logger: { info: jest.fn(), warn: jest.fn(), debug: jest.fn(), error: jest.fn() },+ runAsSystem: mockRunAsSystem,+}));++jest.mock('@librechat/api', () => ({+ AGENT_TRIGGER_SCOPE: 'agent_trigger',}));jest.mock('~/models', () => ({@@ -23,9 +31,18 @@ jest.mock('~/models', () => ({const jwtLogin = require('./jwtStrategy');const { getUserById, updateUser } = require('~/models');-function invokeVerify(payload) {+function request(overrides = {}) {+ return {+ method: 'GET',+ originalUrl: '/api/user',+ headers: {},+ ...overrides,+ };+}++function invokeVerify(payload, req = request()) {return new Promise((resolve, reject) => {- capturedVerifyCallback(payload, (err, user, info) => {+ capturedVerifyCallback(req, payload, (err, user, info) => {if (err) {return reject(err);}@@ -51,6 +68,23 @@ describe('jwtStrategy', () => {expect(user.id).toBe('user-1');expect(user.idOnTheSource).toBeNull();+ expect(capturedStrategyOptions.passReqToCallback).toBe(true);+ });++ it('runs globally scoped authentication reads and writes in system context', async () => {+ getUserById.mockResolvedValue({+ _id: { toString: () => 'user-system' },+ });++ const { user } = await invokeVerify({ id: 'user-system' });++ expect(user).toMatchObject({ id: 'user-system', role: SystemRoles.USER });+ expect(mockRunAsSystem).toHaveBeenCalledTimes(2);+ expect(getUserById).toHaveBeenCalledWith(+ 'user-system',+ '-password -__v -totpSecret -backupCodes +agentTriggerDeletionStartedAt',+ );+ expect(updateUser).toHaveBeenCalledWith('user-system', { role: SystemRoles.USER });});it('preserves a stored idOnTheSource for federated users', async () => {@@ -72,4 +106,81 @@ describe('jwtStrategy', () => {expect(user).toBe(false);});++ it('accepts a scoped trigger token only on an exact marked admission route', async () => {+ getUserById.mockResolvedValue({+ _id: { toString: () => 'user-3' },+ role: SystemRoles.USER,+ });++ const { user } = await invokeVerify(+ { id: 'user-3', scope: 'agent_trigger' },+ request({+ method: 'POST',+ originalUrl: '/trusted/base/api/agents/chat/agents?source=delivery',+ headers: { 'x-lc-agent-trigger': '1' },+ }),+ );++ expect(user.id).toBe('user-3');+ });++ it.each([+ ['an unrelated route', request({ method: 'POST', originalUrl: '/api/user' })],+ [+ 'a missing transport marker',+ request({ method: 'POST', originalUrl: '/api/agents/chat/agents' }),+ ],+ [+ 'a non-POST request',+ request({+ method: 'GET',+ originalUrl: '/api/agents/chat/steer/deliver',+ headers: { 'x-lc-agent-trigger': '1' },+ }),+ ],+ ])('rejects a scoped trigger token on %s', async (_label, req) => {+ const { user, info } = await invokeVerify({ id: 'user-3', scope: 'agent_trigger' }, req);++ expect(user).toBe(false);+ expect(info).toEqual({ message: 'Agent trigger token is not valid for this endpoint' });+ expect(getUserById).not.toHaveBeenCalled();+ });++ it('rejects ordinary and trigger-scoped tokens while account deletion is fenced', async () => {+ getUserById.mockResolvedValue({+ _id: { toString: () => 'user-3' },+ role: SystemRoles.USER,+ agentTriggerDeletionStartedAt: new Date('2026-08-17T12:00:00.000Z'),+ });++ const ordinary = await invokeVerify({ id: 'user-3' });+ const trigger = await invokeVerify(+ { id: 'user-3', scope: 'agent_trigger' },+ request({+ method: 'POST',+ originalUrl: '/api/agents/chat/steer/deliver',+ headers: { 'x-lc-agent-trigger': '1' },+ }),+ );++ expect(ordinary).toEqual({+ user: false,+ info: {+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ },+ });+ expect(trigger).toEqual({+ user: false,+ info: {+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ },+ });+ expect(getUserById).toHaveBeenCalledWith(+ 'user-3',+ '-password -__v -totpSecret -backupCodes +agentTriggerDeletionStartedAt',+ );+ });});diff --git a/api/strategies/openIdJwtStrategy.js b/api/strategies/openIdJwtStrategy.jsindex 3606fe474b..5cd55020f1 100644--- a/api/strategies/openIdJwtStrategy.js+++ b/api/strategies/openIdJwtStrategy.js@@ -1,6 +1,6 @@const cookies = require('cookie');const jwksRsa = require('jwks-rsa');-const { logger, getTenantId } = require('@librechat/data-schemas');+const { logger, getTenantId, runAsSystem } = require('@librechat/data-schemas');const { CacheKeys, SystemRoles } = require('librechat-data-provider');const { Strategy: JwtStrategy, ExtractJwt } = require('passport-jwt');const {@@ -18,7 +18,7 @@ const {getHttpsProxyAgent,math,} = require('@librechat/api');-const { updateUser, findUser } = require('~/models');+const { updateUser, findUser, isAgentTriggerPrincipalActive } = require('~/models');const getLogStores = require('~/cache/getLogStores');const getOpenIdJwtAudience = () => {@@ -173,6 +173,13 @@ const openIdJwtLogin = (openIdConfig) => {if (user) {user.id = user._id.toString();+ if (!(await runAsSystem(() => isAgentTriggerPrincipalActive(user.id)))) {+ done(null, false, {+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ });+ return;+ }/** Absent on the full doc means local user; null skips getUserPrincipals' fallback lookup */user.idOnTheSource ??= null;diff --git a/api/strategies/openIdJwtStrategy.spec.js b/api/strategies/openIdJwtStrategy.spec.jsindex a1993aa5c7..e893628edb 100644--- a/api/strategies/openIdJwtStrategy.spec.js+++ b/api/strategies/openIdJwtStrategy.spec.js@@ -10,6 +10,7 @@ const mockAuthUserDocCacheStore = {};const mockGetLogStores = jest.fn(() => mockAuthUserDocCacheStore);const mockGetTenantId = jest.fn();+const mockRunAsSystem = jest.fn((callback) => callback());jest.mock('passport-jwt', () => ({Strategy: jest.fn((opts, verifyCallback) => {capturedStrategyOptions = opts;@@ -29,6 +30,7 @@ jest.mock('https-proxy-agent', () => ({jest.mock('@librechat/data-schemas', () => ({logger: { info: jest.fn(), warn: jest.fn(), debug: jest.fn(), error: jest.fn() },getTenantId: mockGetTenantId,+ runAsSystem: mockRunAsSystem,}));jest.mock('@librechat/api', () => ({isEnabled: jest.fn(() => false),@@ -48,6 +50,7 @@ jest.mock('@librechat/api', () => ({jest.mock('~/models', () => ({findUser: jest.fn(),updateUser: jest.fn(),+ isAgentTriggerPrincipalActive: jest.fn(() => true),}));jest.mock('~/server/services/Files/strategies', () => ({getStrategyFunctions: jest.fn(() => ({@@ -69,7 +72,7 @@ const {setCachedAuthUserDoc,} = require('@librechat/api');const openIdJwtLogin = require('./openIdJwtStrategy');-const { findUser, updateUser } = require('~/models');+const { findUser, updateUser, isAgentTriggerPrincipalActive } = require('~/models');function resetAuthUserDocCacheMocks() {mockGetTenantId.mockReturnValue(undefined);@@ -87,6 +90,8 @@ function resetAuthUserDocCacheMocks() {beforeEach(() => {resetAuthUserDocCacheMocks();+ mockRunAsSystem.mockClear();+ isAgentTriggerPrincipalActive.mockResolvedValue(true);});function withEnv(env, callback) {@@ -451,6 +456,32 @@ describe('openIdJwtStrategy – auth user document cache', () => {expect(invalidateCachedAuthUserDoc).not.toHaveBeenCalled();});+ it('rejects a cached OpenID user while account deletion is fenced', async () => {+ mockGetTenantId.mockReturnValue('tenant-a');+ getAuthUserDocCacheMode.mockReturnValue('on');+ getCachedAuthUserDoc.mockResolvedValue({+ _id: 'cached-user',+ role: SystemRoles.USER,+ provider: 'openid',+ tenantId: 'tenant-a',+ });+ isAgentTriggerPrincipalActive.mockResolvedValue(false);++ const result = await invokeVerify(req, payload);++ expect(result).toEqual({+ user: false,+ info: {+ message: 'Account deletion is in progress',+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ },+ });+ expect(findOpenIDUser).not.toHaveBeenCalled();+ expect(mockRunAsSystem).toHaveBeenCalledWith(expect.any(Function));+ expect(isAgentTriggerPrincipalActive).toHaveBeenCalledWith('cached-user');+ expect(setCachedAuthUserDoc).not.toHaveBeenCalled();+ });+it('populates the cache after a miss with the fresh user document', async () => {getAuthUserDocCacheMode.mockReturnValue('on');getCachedAuthUserDoc.mockResolvedValue(undefined);diff --git a/config/delete-user.js b/config/delete-user.jsindex 66e325d1ee..fec8df7590 100644--- a/config/delete-user.js+++ b/config/delete-user.js@@ -3,6 +3,7 @@// @ts-nocheckconst path = require('path');const mongoose = require('mongoose');+const { createModels, createMethods, runAsSystem } = require('@librechat/data-schemas');const {Key,User,@@ -27,11 +28,23 @@ const {Transaction,Conversation,ConversationTag,-} = require('@librechat/data-schemas').createModels(mongoose);+} = createModels(mongoose);require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });+const {+ GenerationJobManager,+ createStreamServices,+ waitForKeyvRedisClient,+} = require('@librechat/api');+const getLogStores = require('~/cache/getLogStores');const { askQuestion, silentExit } = require('./helpers');const connect = require('./connect');+const TRIGGER_DRAIN_TIMEOUT_MS = 35_000;+const TRIGGER_DRAIN_POLL_MS = 100;+const methods = createMethods(mongoose, { getCache: getLogStores });++const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));+async function gracefulExit(code = 0) {try {await mongoose.disconnect();@@ -76,41 +89,157 @@ async function gracefulExit(code = 0) {const uid = user._id.toString();- // 5) Build and run deletion tasks- const tasks = [- Action.deleteMany({ user: uid }),- Agent.deleteMany({ author: uid }),- AgentApiKey.deleteMany({ user: uid }),- Assistant.deleteMany({ user: uid }),- Balance.deleteMany({ user: uid }),- ConversationTag.deleteMany({ user: uid }),- Conversation.deleteMany({ user: uid }),- Message.deleteMany({ user: uid }),- File.deleteMany({ user: uid }),- Key.deleteMany({ userId: uid }),- MemoryEntry.deleteMany({ userId: uid }),- PluginAuth.deleteMany({ userId: uid }),- Prompt.deleteMany({ author: uid }),- PromptGroup.deleteMany({ author: uid }),- Preset.deleteMany({ user: uid }),- Session.deleteMany({ user: uid }),- SharedLink.deleteMany({ user: uid }),- ToolCall.deleteMany({ user: uid }),- Token.deleteMany({ userId: uid }),- AclEntry.deleteMany({ principalId: user._id }),- ];-- if (deleteTx) {- tasks.push(Transaction.deleteMany({ user: uid }));+ // The CLI can coordinate live generation aborts only through the shared+ // Redis stream store. Without it, require an explicit offline assertion.+ let streamServices;+ try {+ await waitForKeyvRedisClient();+ streamServices = createStreamServices();+ } catch (error) {+ console.yellow(+ `Shared Redis generation coordination is unreachable: ${error instanceof Error ? error.message : String(error)}`,+ );+ streamServices = createStreamServices({ useRedis: false });}+ const hasSharedGenerationStore = streamServices.isRedis;+ let allProcessesStopped = false;+ if (!hasSharedGenerationStore) {+ const confirmOffline = await askQuestion(+ 'Shared Redis generation coordination is unavailable. Confirm ALL LibreChat app, worker, and other deletion CLI processes are stopped before continuing. (y/N)',+ );+ if (confirmOffline.toLowerCase() !== 'y') {+ console.yellow('Aborted. Stop every LibreChat process or enable Redis stream coordination.');+ return gracefulExit(1);+ }+ allProcessesStopped = true;+ } else {+ GenerationJobManager.configure({ ...streamServices, cleanupOnComplete: false });+ GenerationJobManager.initialize();+ }++ let deletionFence;+ let userDeleted = false;++ try {+ deletionFence = new Date();+ let fenceState = await runAsSystem(() =>+ methods.beginAgentTriggerUserDeletion(uid, deletionFence),+ );+ if (fenceState === 'in_progress') {+ deletionFence = undefined;+ if (!allProcessesStopped) {+ const confirmRecovery = await askQuestion(+ 'An account-deletion fence already exists. Confirm ALL LibreChat app, worker, and other deletion CLI processes are stopped to recover it only if stale. (y/N)',+ );+ if (confirmRecovery.toLowerCase() !== 'y') {+ throw new Error('Account deletion is already in progress');+ }+ allProcessesStopped = true;+ }++ const recoveredAt = new Date();+ fenceState = await runAsSystem(() =>+ methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt),+ );+ if (fenceState !== 'acquired') {+ throw new Error(+ fenceState === 'missing'+ ? 'User disappeared before stale-fence recovery'+ : 'Account deletion is active or its fence is not stale enough to recover',+ );+ }+ deletionFence = recoveredAt;+ }+ if (fenceState === 'missing') {+ deletionFence = undefined;+ }++ if (deletionFence != null) {+ await runAsSystem(() =>+ methods.prepareAgentTriggerUserPurge(uid, deletionFence, user.tenantId),+ );+ if (hasSharedGenerationStore) {+ const deadline = Date.now() + TRIGGER_DRAIN_TIMEOUT_MS;+ while (+ (await runAsSystem(() =>+ methods.countActiveAgentTriggerDeliveriesByUser(uid, new Date()),+ )) > 0+ ) {+ if (Date.now() >= deadline) {+ throw new Error('Timed out draining active agent trigger deliveries');+ }+ await delay(TRIGGER_DRAIN_POLL_MS);+ }+ }+ }++ if (hasSharedGenerationStore) {+ const cleanupBlockingAgentRuns = await GenerationJobManager.getCleanupBlockingJobIdsForUser(+ uid,+ user.tenantId,+ );+ await Promise.all(+ cleanupBlockingAgentRuns.map((streamId) =>+ GenerationJobManager.abortJob(streamId, { awaitProviderDrain: true }),+ ),+ );+ }- await Promise.all(tasks);+ // 5) Build and run deletion tasks+ const tasks = [+ Action.deleteMany({ user: uid }),+ Agent.deleteMany({ author: uid }),+ AgentApiKey.deleteMany({ user: uid }),+ Assistant.deleteMany({ user: uid }),+ Balance.deleteMany({ user: uid }),+ ConversationTag.deleteMany({ user: uid }),+ Conversation.deleteMany({ user: uid }),+ Message.deleteMany({ user: uid }),+ File.deleteMany({ user: uid }),+ Key.deleteMany({ userId: uid }),+ MemoryEntry.deleteMany({ userId: uid }),+ PluginAuth.deleteMany({ userId: uid }),+ Prompt.deleteMany({ author: uid }),+ PromptGroup.deleteMany({ author: uid }),+ Preset.deleteMany({ user: uid }),+ Session.deleteMany({ user: uid }),+ SharedLink.deleteMany({ user: uid }),+ ToolCall.deleteMany({ user: uid }),+ Token.deleteMany({ userId: uid }),+ AclEntry.deleteMany({ principalId: user._id }),+ ];- // 6) Remove user from all groups- await Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });+ if (deleteTx) {+ tasks.push(Transaction.deleteMany({ user: uid }));+ }- // 7) Finally delete the user document itself- await User.deleteOne({ _id: uid });+ await Promise.all(tasks);++ // 6) Remove user from all groups+ await Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });++ // 7) Finally delete the user document itself+ const deletedUser = await runAsSystem(() => methods.deleteUserById(uid));+ if (deletedUser.deletedCount !== 1) {+ throw new Error('User disappeared before account deletion could commit');+ }+ userDeleted = true;+ await runAsSystem(() => methods.deleteAgentTriggerDeliveriesByUser(uid));+ } finally {+ if (deletionFence != null && !userDeleted) {+ await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch(+ (error) => console.error('Failed to disarm trigger purge recovery:', error),+ );+ await runAsSystem(() => methods.cancelAgentTriggerUserDeletion(uid, deletionFence)).catch(+ (error) => console.error('Failed to release account-deletion fence:', error),+ );+ }+ if (hasSharedGenerationStore) {+ await GenerationJobManager.destroy().catch((error) =>+ console.error('Failed to close generation coordination:', error),+ );+ }+ }console.green(`✔ Successfully deleted user ${email} and all associated data.`);if (!deleteTx) {diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.tsindex 1d0e5fbac8..e7cae56ad4 100644--- a/packages/api/src/admin/users.spec.ts+++ b/packages/api/src/admin/users.spec.ts@@ -53,6 +53,12 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {return {findUsers: jest.fn().mockResolvedValue([]),countUsers: jest.fn().mockResolvedValue(0),+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),+ cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),+ drainAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined),+ prepareAgentTriggerUserPurge: jest.fn().mockResolvedValue(undefined),+ cancelAgentTriggerUserPurge: jest.fn().mockResolvedValue(true),+ purgeAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined),deleteUserById: jest.fn().mockResolvedValue({ deletedCount: 1, message: 'User was deleted successfully.' }),@@ -340,6 +346,19 @@ describe('createAdminUsersHandlers', () => {expect(status).toHaveBeenCalledWith(200);expect(json).toHaveBeenCalledWith({ message: 'User was deleted successfully.' });+ expect(deps.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(+ validUserId,+ expect.any(Date),+ );+ expect(deps.drainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId);+ expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(+ validUserId,+ expect.any(Date),+ undefined,+ );+ expect(deps.purgeAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId);+ expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();});it('returns fallback message when result.message is empty', async () => {@@ -464,6 +483,12 @@ describe('createAdminUsersHandlers', () => {expect(status).toHaveBeenCalledWith(404);expect(deps.deleteConfig).not.toHaveBeenCalled();expect(deps.deleteAclEntries).not.toHaveBeenCalled();+ expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(+ validUserId,+ expect.any(Date),+ );+ expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));});it('returns 400 for invalid ObjectId', async () => {@@ -500,6 +525,84 @@ describe('createAdminUsersHandlers', () => {expect(status).toHaveBeenCalledWith(500);expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });+ expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(+ validUserId,+ expect.any(Date),+ );+ expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));+ expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();+ });++ it('does not delete while another deletion owns the trigger fence', async () => {+ const deps = createDeps({+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('in_progress'),+ });+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ expect(status).toHaveBeenCalledWith(409);+ expect(json).toHaveBeenCalledWith({ error: 'User deletion is already in progress' });+ expect(deps.drainAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();+ expect(deps.deleteUserById).not.toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();+ });++ it('returns not found without draining when the trigger fence principal is missing', async () => {+ const deps = createDeps({+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('missing'),+ });+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ expect(status).toHaveBeenCalledWith(404);+ expect(json).toHaveBeenCalledWith({ error: 'User not found' });+ expect(deps.drainAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();+ expect(deps.deleteUserById).not.toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();+ });++ it('drains before commit and purges only after the user is deleted', async () => {+ const deps = createDeps();+ const handlers = createAdminUsersHandlers(deps);+ const { req, res } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ const beginOrder = (deps.beginAgentTriggerUserDeletion as jest.Mock).mock+ .invocationCallOrder[0];+ const prepareOrder = (deps.prepareAgentTriggerUserPurge as jest.Mock).mock+ .invocationCallOrder[0];+ const drainOrder = (deps.drainAgentTriggerDeliveriesForUser as jest.Mock).mock+ .invocationCallOrder[0];+ const deleteOrder = (deps.deleteUserById as jest.Mock).mock.invocationCallOrder[0];+ const purgeOrder = (deps.purgeAgentTriggerDeliveriesForUser as jest.Mock).mock+ .invocationCallOrder[0];+ expect(beginOrder).toBeLessThan(drainOrder);+ expect(beginOrder).toBeLessThan(prepareOrder);+ expect(prepareOrder).toBeLessThan(drainOrder);+ expect(drainOrder).toBeLessThan(deleteOrder);+ expect(deleteOrder).toBeLessThan(purgeOrder);+ });++ it('leaves durable purge recovery armed when immediate post-commit cleanup fails', async () => {+ const deps = createDeps({+ purgeAgentTriggerDeliveriesForUser: jest.fn().mockRejectedValue(new Error('db down')),+ });+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });++ await handlers.deleteUser(req, res);++ expect(status).toHaveBeenCalledWith(500);+ expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });+ expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId);+ expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled();+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();});});});diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.tsindex d24eb1b759..5a46d3c748 100644--- a/packages/api/src/admin/users.ts+++ b/packages/api/src/admin/users.ts@@ -24,11 +24,25 @@ export interface AdminUsersDeps {options?: { limit?: number; offset?: number; sort?: Record<string, 1 | -1> },) => Promise<IUser[]>;countUsers: (filter?: FilterQuery<IUser>) => Promise<number>;+ beginAgentTriggerUserDeletion: (+ userId: string,+ startedAt: Date,+ ) => Promise<'acquired' | 'in_progress' | 'missing'>;+ cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;+ drainAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;+ prepareAgentTriggerUserPurge: (+ userId: string,+ fenceStartedAt: Date,+ tenantId?: string,+ ) => Promise<void>;+ cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;+ purgeAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;/*** Thin data-layer delete — removes the User document only.* Full cascade of user-owned resources (conversations, messages, files, tokens, etc.)* is handled by `UserController.deleteUserController` in the self-delete flow.- * This admin endpoint currently cascades Config and AclEntries.+ * This admin endpoint fences durable triggers around the user commit and currently+ * cascades Config and AclEntries.* A future iteration should consolidate the full cascade into a shared service function.*/deleteUserById: (userId: string) => Promise<UserDeleteResult>;@@ -47,7 +61,19 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {searchUsers: (req: ServerRequest, res: Response) => Promise<Response>;deleteUser: (req: ServerRequest, res: Response) => Promise<Response>;} {- const { findUsers, countUsers, deleteUserById, deleteConfig, deleteAclEntries } = deps;+ const {+ findUsers,+ countUsers,+ beginAgentTriggerUserDeletion,+ cancelAgentTriggerUserDeletion,+ drainAgentTriggerDeliveriesForUser,+ prepareAgentTriggerUserPurge,+ cancelAgentTriggerUserPurge,+ purgeAgentTriggerDeliveriesForUser,+ deleteUserById,+ deleteConfig,+ deleteAclEntries,+ } = deps;async function listUsersHandler(req: ServerRequest, res: Response) {try {@@ -126,8 +152,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {}async function deleteUserHandler(req: ServerRequest, res: Response) {+ let targetUserId: string | undefined;+ let triggerDeletionFence: Date | undefined;+ let userDeleted = false;+try {const { id } = req.params as { id: string };+ targetUserId = id;if (!isValidObjectIdString(id)) {return res.status(400).json({ error: 'Invalid user ID format' });@@ -138,7 +169,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {return res.status(403).json({ error: 'Cannot delete your own account' });}- const [targetUser] = await findUsers({ _id: id }, 'role', { limit: 1 });+ const [targetUser] = await findUsers({ _id: id }, 'role tenantId', { limit: 1 });if (targetUser?.role === SystemRoles.ADMIN) {const adminCount = await countUsers({ role: SystemRoles.ADMIN });if (adminCount <= 1) {@@ -146,11 +177,29 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {}}+ triggerDeletionFence = new Date();+ const fenceState = await beginAgentTriggerUserDeletion(id, triggerDeletionFence);+ if (fenceState === 'in_progress') {+ triggerDeletionFence = undefined;+ return res.status(409).json({ error: 'User deletion is already in progress' });+ }+ if (fenceState === 'missing') {+ triggerDeletionFence = undefined;+ return res.status(404).json({ error: 'User not found' });+ }+ await prepareAgentTriggerUserPurge(id, triggerDeletionFence, targetUser?.tenantId);+ await drainAgentTriggerDeliveriesForUser(id);+const result = await deleteUserById(id);if (result.deletedCount === 0) {+ await cancelAgentTriggerUserPurge(id, triggerDeletionFence);+ await cancelAgentTriggerUserDeletion(id, triggerDeletionFence);+ triggerDeletionFence = undefined;return res.status(404).json({ error: 'User not found' });}+ userDeleted = true;+ await purgeAgentTriggerDeliveriesForUser(id);if (targetUser?.role === SystemRoles.ADMIN) {const remaining = await countUsers({ role: SystemRoles.ADMIN });@@ -175,6 +224,18 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {return res.status(200).json({ message: result.message || 'User deleted successfully' });} catch (error) {+ if (targetUserId != null && triggerDeletionFence != null && !userDeleted) {+ try {+ await cancelAgentTriggerUserPurge(targetUserId, triggerDeletionFence);+ } catch (purgeFenceError) {+ logger.error('[adminUsers] failed to disarm trigger purge recovery:', purgeFenceError);+ }+ try {+ await cancelAgentTriggerUserDeletion(targetUserId, triggerDeletionFence);+ } catch (fenceError) {+ logger.error('[adminUsers] failed to release trigger deletion fence:', fenceError);+ }+ }logger.error('[adminUsers] deleteUser error:', error);return res.status(500).json({ error: 'Failed to delete user' });}diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.mdnew file mode 100644index 0000000000..1da00fe38e--- /dev/null+++ b/packages/api/src/agents/triggers/README.md@@ -0,0 +1,67 @@+# Agent trigger delivery++This module is the trusted, source-neutral boundary for asynchronous agent work. A schedule,+webhook, queue consumer, MCP integration, or internal event adapter produces the same versioned+envelope and calls `enqueueAgentTrigger`; the adapter does not invoke an agent runtime directly.++## Adapter contract++- Authenticate and authorize the source before creating an envelope.+- Strip credentials and transport secrets from `event.payload`.+- Give each source event a stable `event.id`, and keep `deliveryId` stable for retries to one+ target. A retry may use a fresh `requestId` and `receivedAt`.+- Render bounded model input on the host. Infrastructure and routing remain server-controlled.+- Use `orderingKey` only when deliveries must remain ordered across different event sources.+ Without an override, ordering is scoped to the user, source, mode, agent, and conversation.++```js+const { createAgentTriggerEnvelope } = require('@librechat/api');+const { enqueueAgentTrigger } = require('~/server/services/Agents/triggers');++await enqueueAgentTrigger(+ createAgentTriggerEnvelope({+ mode: 'fire',+ requestId,+ deliveryId,+ receivedAt: Date.now(),+ principal: { id: userId, role, tenantId },+ event: {+ id: eventId,+ type: 'resource.ready',+ occurredAt,+ source: { id: webhookId, type: 'webhook' },+ payload: sanitizedPayload,+ },+ target: { agentId },+ input,+ }),+ { orderingKey: resourceId },+);+```++## Guarantees++- Mongo owns queue state, leases, retry history, and dead letters across restarts and replicas.+- A fresh token fences every claim, including reclaims by the same process.+- A delivery is at-least-once. Fire and steer admission reuse the envelope's stable idempotency+ identity, so ambiguous retries do not duplicate accepted work.+- Retryable failures use bounded exponential backoff and honor `Retry-After`. Invalid envelopes,+ permanent authorization failures, and exhausted retries become durable dead letters.+- Matching ordering lanes serialize sequence allocation and queue publication behind a+ Mongo-fenced publisher. A staging row is durable before taking that fence, and any replica can+ finish an abandoned publication before allocating the next sequence, so a later delivery can+ never overtake the invisible gap. Dead letters are terminal and do not block later work; an+ explicit requeue admits the dead letter as a new lane tail so it cannot overlap newer in-flight+ work. Inactive lane counters are reclaimed once no staging, queued, leased, or dead delivery+ remains.+- Successful records expire after 90 days. Dead letters remain until explicitly requeued or+ removed. Account deletion first fences admission and drains active leases without destroying+ queued work. A delivery deferred by that fence releases its lease and restores the attempt it+ reserved, so a rolled-back deletion cannot exhaust its retry budget. Before deleting the user,+ every deletion path durably arms an exact-fence purge marker; payloads are purged only after the+ user deletion commits, and every replica retries any orphaned post-commit marker until cleanup+ succeeds. An abandoned deletion fence can be recovered through `config/delete-user.js` only+ after an operator confirms every competing app, worker, and deletion CLI process is stopped.++`getAgentTriggerDeadLetters` and `requeueAgentTrigger` are intentionally trusted in-process+operations. Exposing them through an admin API requires a separate authorization and audit layer.diff --git a/packages/api/src/agents/triggers/delivery.integration.spec.ts b/packages/api/src/agents/triggers/delivery.integration.spec.tsnew file mode 100644index 0000000000..6bdf157c95--- /dev/null+++ b/packages/api/src/agents/triggers/delivery.integration.spec.ts@@ -0,0 +1,102 @@+import mongoose from 'mongoose';+import { MongoMemoryServer } from 'mongodb-memory-server';+import { createMethods, createModels } from '@librechat/data-schemas';+import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service';+import type { AgentTriggerFetch } from './host';+import { __resetShutdownStateForTests } from '../../app/shutdown';+import { createAgentTriggerEnvelope } from './envelope';+import { createAgentTriggerService } from './service';++const DB_SETUP_TIMEOUT_MS = 60_000;+let mongoServer: MongoMemoryServer;+let service: AgentTriggerService | undefined;++beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri(), { autoIndex: false });+ createModels(mongoose);+}, DB_SETUP_TIMEOUT_MS);++afterEach(async () => {+ await service?.stop();+ service = undefined;+ __resetShutdownStateForTests();+ await mongoose.models.AgentTriggerDelivery.deleteMany({});+});++afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+}, DB_SETUP_TIMEOUT_MS);++const envelope = () =>+ createAgentTriggerEnvelope({+ mode: 'fire',+ requestId: 'request-1',+ deliveryId: 'delivery-1',+ receivedAt: 20,+ principal: { id: new mongoose.Types.ObjectId().toString(), tenantId: 'tenant-1' },+ target: { agentId: 'agent-1' },+ event: {+ id: 'event-1',+ type: 'resource.ready',+ occurredAt: 10,+ source: { id: 'source-1', type: 'webhook' },+ },+ input: 'Handle the ready resource.',+ });++async function eventuallySucceeded(deliveryKey: string) {+ const deadline = Date.now() + 5_000;+ while (Date.now() < deadline) {+ const delivery = await service!.getDelivery(deliveryKey);+ if (delivery?.status === 'succeeded') {+ return delivery;+ }+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ throw new Error('Timed out waiting for the queued trigger to succeed');+}++describe('durable trigger delivery integration', () => {+ it('moves a trusted envelope through Mongo, the lease worker, and host admission', async () => {+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(+ async () =>+ new Response(+ JSON.stringify({+ status: 'started',+ streamId: 'conversation-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 25,+ }),+ { status: 200, headers: { 'content-type': 'application/json' } },+ ),+ );+ service = createAgentTriggerService({+ methods: createMethods(mongoose) as ReturnType<typeof createMethods> &+ AgentTriggerDeliveryPersistence,+ fetch: fetcher,+ mintToken: () => 'trigger-token',+ deliveryOptions: { concurrency: 2, tickMs: 5, retryBaseMs: 5 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });++ const receipt = await service.enqueue(envelope(), { orderingKey: 'resource-1' });+ const delivered = await eventuallySucceeded(receipt.deliveryKey);++ expect(fetcher).toHaveBeenCalledTimes(1);+ expect(delivered).toMatchObject({+ id: receipt.id,+ status: 'succeeded',+ attempts: 1,+ result: {+ mode: 'fire',+ status: 'started',+ conversationId: 'conversation-1',+ },+ history: [{ attempt: 1, outcome: 'succeeded' }],+ });+ });+});diff --git a/packages/api/src/agents/triggers/delivery.spec.ts b/packages/api/src/agents/triggers/delivery.spec.tsnew file mode 100644index 0000000000..022c362e7c--- /dev/null+++ b/packages/api/src/agents/triggers/delivery.spec.ts@@ -0,0 +1,108 @@+import {+ AgentTriggerDeliveryError,+ MAX_AGENT_TRIGGER_ENVELOPE_BYTES,+ prepareAgentTriggerDelivery,+} from './delivery';+import { createAgentTriggerEnvelope } from './envelope';++const envelope = (overrides: Record<string, unknown> = {}) =>+ createAgentTriggerEnvelope({+ mode: 'fire',+ requestId: 'request-1',+ deliveryId: 'delivery-1',+ receivedAt: 20,+ principal: { id: '507f1f77bcf86cd799439011', tenantId: 'tenant-1' },+ target: { agentId: 'agent-1' },+ event: {+ id: 'event-1',+ type: 'resource.ready',+ occurredAt: 10,+ source: { id: 'source-1', type: 'webhook' },+ payload: { b: 2, a: 1 },+ },+ input: 'Handle the ready resource.',+ ...overrides,+ });++describe('prepareAgentTriggerDelivery', () => {+ it('prepares a bounded persistent record with a default ordering lane', () => {+ const availableAt = new Date('2026-08-17T12:00:00.000Z');+ const prepared = prepareAgentTriggerDelivery(envelope(), { availableAt });++ expect(prepared).toMatchObject({+ deliveryKey: expect.stringMatching(/^trigger_[a-f0-9]{64}$/),+ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/),+ orderingKey: expect.stringMatching(/^trigger_lane_[a-f0-9]{64}$/),+ user: '507f1f77bcf86cd799439011',+ tenantId: 'tenant-1',+ availableAt,+ });+ expect(prepared.availableAt).not.toBe(availableAt);+ });++ it('treats fresh ingress metadata and payload key order as the same retry', () => {+ const first = prepareAgentTriggerDelivery(envelope());+ const retried = prepareAgentTriggerDelivery(+ envelope({+ requestId: 'request-2',+ receivedAt: 30,+ event: {+ id: 'event-1',+ type: 'resource.ready',+ occurredAt: 10,+ source: { id: 'source-1', type: 'webhook' },+ payload: { a: 1, b: 2 },+ },+ }),+ );++ expect(retried.deliveryKey).toBe(first.deliveryKey);+ expect(retried.fingerprint).toBe(first.fingerprint);+ expect(retried.orderingKey).toBe(first.orderingKey);+ });++ it('detects execution-content changes under the same delivery identity', () => {+ const first = prepareAgentTriggerDelivery(envelope());+ const changed = prepareAgentTriggerDelivery(envelope({ input: 'Do something else.' }));++ expect(changed.deliveryKey).toBe(first.deliveryKey);+ expect(changed.fingerprint).not.toBe(first.fingerprint);+ });++ it('supports an explicit cross-source ordering lane scoped to the principal', () => {+ const first = prepareAgentTriggerDelivery(envelope(), { orderingKey: 'game-42' });+ const second = prepareAgentTriggerDelivery(+ envelope({+ deliveryId: 'delivery-2',+ event: {+ id: 'event-2',+ type: 'clock.tick',+ occurredAt: 11,+ source: { id: 'source-2', type: 'scheduler' },+ },+ }),+ { orderingKey: ' game-42 ' },+ );+ const anotherUser = prepareAgentTriggerDelivery(+ envelope({ principal: { id: '507f1f77bcf86cd799439012', tenantId: 'tenant-1' } }),+ { orderingKey: 'game-42' },+ );++ expect(second.orderingKey).toBe(first.orderingKey);+ expect(anotherUser.orderingKey).not.toBe(first.orderingKey);+ });++ it('rejects invalid scheduling metadata and oversized envelopes', () => {+ expect(() =>+ prepareAgentTriggerDelivery(envelope(), { availableAt: new Date(Number.NaN) }),+ ).toThrow(AgentTriggerDeliveryError);+ expect(() => prepareAgentTriggerDelivery(envelope(), { orderingKey: ' ' })).toThrow(+ AgentTriggerDeliveryError,+ );+ expect(() =>+ prepareAgentTriggerDelivery(+ envelope({ input: 'x'.repeat(MAX_AGENT_TRIGGER_ENVELOPE_BYTES) }),+ ),+ ).toThrow(`Agent trigger envelope exceeds ${MAX_AGENT_TRIGGER_ENVELOPE_BYTES} bytes`);+ });+});diff --git a/packages/api/src/agents/triggers/delivery.ts b/packages/api/src/agents/triggers/delivery.tsnew file mode 100644index 0000000000..5d511b263e--- /dev/null+++ b/packages/api/src/agents/triggers/delivery.ts@@ -0,0 +1,115 @@+import { createHash } from 'node:crypto';+import type { AgentTriggerEnvelope } from './envelope';+import { getAgentTriggerIdempotencyKey, parseAgentTriggerEnvelope } from './envelope';++export const MAX_AGENT_TRIGGER_ENVELOPE_BYTES: number = 1024 * 1024;++export interface AgentTriggerEnqueueOptions {+ /** Trusted lane override. Matching lanes dispatch strictly in enqueue order. */+ orderingKey?: string;+ /** Delays first eligibility without changing source occurrence time. */+ availableAt?: Date;+}++export interface PreparedAgentTriggerDelivery {+ deliveryKey: string;+ fingerprint: string;+ orderingKey: string;+ envelope: AgentTriggerEnvelope;+ user: string;+ tenantId?: string;+ availableAt: Date;+}++export class AgentTriggerDeliveryError extends TypeError {+ constructor(message: string) {+ super(message);+ this.name = 'AgentTriggerDeliveryError';+ }+}++function canonicalJson(value: unknown): string {+ if (value === null || typeof value === 'boolean' || typeof value === 'number') {+ return JSON.stringify(value);+ }+ if (typeof value === 'string') {+ return JSON.stringify(value);+ }+ if (Array.isArray(value)) {+ return `[${value.map(canonicalJson).join(',')}]`;+ }+ const record = value as Record<string, unknown>;+ return `{${Object.keys(record)+ .sort()+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)+ .join(',')}}`;+}++function digest(value: unknown): string {+ return createHash('sha256').update(canonicalJson(value)).digest('hex');+}++function requireAvailableAt(value: Date | undefined): Date {+ const availableAt = value ?? new Date();+ if (!(availableAt instanceof Date) || !Number.isFinite(availableAt.getTime())) {+ throw new AgentTriggerDeliveryError('availableAt must be a valid Date');+ }+ return new Date(availableAt);+}++function orderingIdentity(+ envelope: AgentTriggerEnvelope,+ configuredKey: string | undefined,+): string {+ let lane: unknown;+ if (configuredKey != null) {+ const trimmed = configuredKey.trim();+ if (trimmed.length === 0 || trimmed.length > 256) {+ throw new AgentTriggerDeliveryError('orderingKey must contain between 1 and 256 characters');+ }+ lane = ['explicit', trimmed];+ } else {+ lane = [+ 'default',+ envelope.event.source.type,+ envelope.event.source.id,+ envelope.mode,+ envelope.target.agentId,+ envelope.mode === 'steer' ? envelope.target.conversationId : '',+ ];+ }+ return `trigger_lane_${digest([+ envelope.principal.tenantId ?? '',+ envelope.principal.userId,+ lane,+ ])}`;+}++/** Validates, bounds, detaches, and fingerprints one durable source delivery. */+export function prepareAgentTriggerDelivery(+ value: unknown,+ options: AgentTriggerEnqueueOptions = {},+): PreparedAgentTriggerDelivery {+ const envelope = parseAgentTriggerEnvelope(value);+ const serialized = canonicalJson(envelope);+ const bytes = Buffer.byteLength(serialized, 'utf8');+ if (bytes > MAX_AGENT_TRIGGER_ENVELOPE_BYTES) {+ throw new AgentTriggerDeliveryError(+ `Agent trigger envelope exceeds ${MAX_AGENT_TRIGGER_ENVELOPE_BYTES} bytes`,+ );+ }++ // requestId and receivedAt identify an ingress attempt, not the logical+ // source-event-to-target delivery. Excluding them lets a retried ingress use+ // a fresh request trace without weakening content-conflict detection.+ const { requestId: _requestId, receivedAt: _receivedAt, ...durableIdentity } = envelope;+ return {+ deliveryKey: getAgentTriggerIdempotencyKey(envelope),+ fingerprint: digest(durableIdentity),+ orderingKey: orderingIdentity(envelope, options.orderingKey),+ envelope,+ user: envelope.principal.userId,+ ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),+ availableAt: requireAvailableAt(options.availableAt),+ };+}diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.tsnew file mode 100644index 0000000000..4a0eaff1f5--- /dev/null+++ b/packages/api/src/agents/triggers/engine.spec.ts@@ -0,0 +1,576 @@+import type { AgentTriggerDeliveryRecord, AgentTriggerDeliveryStore } from './engine';+import type { AgentTriggerExecutionResult } from './host';+import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine';+import { AgentTriggerDispatchError } from './dispatch';+import { AgentTriggerExecutionError } from './host';++jest.mock('@librechat/data-schemas', () => {+ const actual = jest.requireActual('@librechat/data-schemas');+ return {+ ...actual,+ logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+});++const START = new Date('2026-08-17T12:00:00.000Z');++const successResult = (): AgentTriggerExecutionResult => ({+ mode: 'fire',+ status: 'started',+ conversationId: 'conversation-1',+});++function delivery(overrides: Partial<AgentTriggerDeliveryRecord> = {}): AgentTriggerDeliveryRecord {+ return {+ id: 'delivery-row-1',+ user: 'user-1',+ claimToken: 'claim-1',+ deliveryKey: 'trigger_1',+ fingerprint: 'fingerprint-1',+ orderingKey: 'ordering-1',+ laneSequence: 1,+ envelope: { version: 1 },+ status: 'leased',+ attempts: 0,+ availableAt: START,+ createdAt: START,+ leaseBy: 'worker-1',+ leaseUntil: new Date(START.getTime() + 120_000),+ ...overrides,+ };+}++function storeWith(overrides: Partial<AgentTriggerDeliveryStore> = {}): AgentTriggerDeliveryStore {+ return {+ claimNext: jest.fn(async () => delivery()),+ findEarlierUnsettled: jest.fn(async () => null),+ release: jest.fn(async () => true),+ beginAttempt: jest.fn(async () => 1),+ defer: jest.fn(async () => true),+ complete: jest.fn(async () => true),+ retry: jest.fn(async () => true),+ dead: jest.fn(async () => true),+ ...overrides,+ };+}++describe('createAgentTriggerDeliveryEngine', () => {+ it('claims, dispatches, and completes with the per-claim fence', async () => {+ const store = storeWith();+ const dispatch = jest.fn(async () => successResult());+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START, workerId: 'worker-1' },+ { concurrency: 1 },+ );++ await expect(engine.runTick()).resolves.toBe(1);++ expect(dispatch).toHaveBeenCalledWith({ version: 1 }, { signal: expect.any(AbortSignal) });+ expect(store.beginAttempt).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ });+ expect(store.complete).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: 1,+ result: successResult(),+ settledAt: START,+ });+ });++ it('cancels and drains an in-flight delivery for one user', async () => {+ const store = storeWith();+ let dispatchStarted: (() => void) | undefined;+ const started = new Promise<void>((resolve) => {+ dispatchStarted = resolve;+ });+ const dispatch = jest.fn(+ async (_envelope: unknown, options?: { signal?: AbortSignal }) =>+ new Promise<AgentTriggerExecutionResult>((_resolve, reject) => {+ dispatchStarted?.();+ options?.signal?.addEventListener(+ 'abort',+ () =>+ reject(+ new AgentTriggerExecutionError('cancelled', {+ mode: 'fire',+ certainty: 'ambiguous',+ retryable: true,+ code: 'ABORTED',+ }),+ ),+ { once: true },+ );+ }),+ );+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START, workerId: 'worker-1' },+ { concurrency: 1, maxAttempts: 1 },+ );++ const tick = engine.runTick();+ await started;+ await engine.cancelUser('user-1');+ await tick;++ expect(store.defer).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: 1,+ availableAt: new Date(START.getTime() + 5_000),+ });+ expect(store.retry).not.toHaveBeenCalled();+ expect(store.dead).not.toHaveBeenCalled();+ expect(store.complete).not.toHaveBeenCalled();+ });++ it('releases a claimed delivery without dispatch after its user is fenced', async () => {+ const store = storeWith();+ const dispatch = jest.fn(async () => successResult());+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START, workerId: 'worker-1' },+ { concurrency: 1 },+ );++ await engine.cancelUser('user-1');+ await expect(engine.runTick()).resolves.toBe(1);++ expect(store.release).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ availableAt: START,+ });+ expect(store.beginAttempt).not.toHaveBeenCalled();+ expect(dispatch).not.toHaveBeenCalled();++ engine.releaseUserCancellation('user-1');+ await engine.runTick();+ expect(dispatch).toHaveBeenCalledTimes(1);+ });++ it('honors Retry-After for a retryable admission rejection', async () => {+ const store = storeWith();+ const error = new AgentTriggerExecutionError('busy', {+ mode: 'fire',+ certainty: 'definite',+ retryable: true,+ code: 'RATE_LIMITED',+ status: 429,+ retryAfter: '120',+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: async () => Promise.reject(error), now: () => START },+ { concurrency: 1 },+ );++ await engine.runTick();++ expect(store.retry).toHaveBeenCalledWith(+ expect.objectContaining({+ claimToken: 'claim-1',+ availableAt: new Date(START.getTime() + 120_000),+ error: expect.objectContaining({+ code: 'RATE_LIMITED',+ certainty: 'definite',+ retryable: true,+ status: 429,+ }),+ }),+ );+ expect(store.dead).not.toHaveBeenCalled();+ });++ it('defers a fenced principal without consuming its delivery attempt', async () => {+ const store = storeWith();+ const engine = createAgentTriggerDeliveryEngine(+ {+ store,+ dispatch: async () =>+ Promise.reject(new AgentTriggerDeliveryDeferredError('principal fenced')),+ now: () => START,+ workerId: 'worker-1',+ },+ { concurrency: 1 },+ );++ await engine.runTick();++ expect(store.defer).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: 1,+ availableAt: new Date(START.getTime() + 5_000),+ });+ expect(store.retry).not.toHaveBeenCalled();+ expect(store.dead).not.toHaveBeenCalled();+ });++ it('defers a cross-replica account-deletion rejection without consuming its attempt', async () => {+ const store = storeWith();+ const engine = createAgentTriggerDeliveryEngine(+ {+ store,+ dispatch: async () =>+ Promise.reject(+ new AgentTriggerExecutionError('account deletion is in progress', {+ mode: 'fire',+ certainty: 'definite',+ retryable: false,+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ status: 409,+ }),+ ),+ now: () => START,+ workerId: 'worker-1',+ },+ { concurrency: 1, maxAttempts: 1 },+ );++ await engine.runTick();++ expect(store.defer).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: 1,+ availableAt: new Date(START.getTime() + 5_000),+ });+ expect(store.retry).not.toHaveBeenCalled();+ expect(store.dead).not.toHaveBeenCalled();+ });++ it('defers an auth-stage account-deletion rejection without consuming its attempt', async () => {+ const store = storeWith();+ const engine = createAgentTriggerDeliveryEngine(+ {+ store,+ dispatch: async () =>+ Promise.reject(+ new AgentTriggerExecutionError('account deletion is in progress', {+ mode: 'fire',+ certainty: 'definite',+ retryable: false,+ code: 'ACCOUNT_DELETION_IN_PROGRESS',+ status: 401,+ }),+ ),+ now: () => START,+ workerId: 'worker-1',+ },+ { concurrency: 1, maxAttempts: 1 },+ );++ await engine.runTick();++ expect(store.defer).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: 1,+ availableAt: new Date(START.getTime() + 5_000),+ });+ expect(store.retry).not.toHaveBeenCalled();+ expect(store.dead).not.toHaveBeenCalled();+ });++ it('does not shorten Retry-After to the exponential backoff cap', async () => {+ const store = storeWith();+ const error = new AgentTriggerExecutionError('maintenance', {+ mode: 'fire',+ certainty: 'definite',+ retryable: true,+ retryAfter: '3600',+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: async () => Promise.reject(error), now: () => START },+ { concurrency: 1, retryCapMs: 1_000 },+ );++ await engine.runTick();++ expect(store.retry).toHaveBeenCalledWith(+ expect.objectContaining({+ availableAt: new Date(START.getTime() + 60 * 60_000),+ }),+ );+ });++ it('dead-letters invalid envelopes without retrying', async () => {+ const store = storeWith();+ const engine = createAgentTriggerDeliveryEngine(+ {+ store,+ dispatch: async () => Promise.reject(new AgentTriggerDispatchError('invalid envelope')),+ now: () => START,+ },+ { concurrency: 1 },+ );++ await engine.runTick();++ expect(store.dead).toHaveBeenCalledWith(+ expect.objectContaining({+ claimToken: 'claim-1',+ error: expect.objectContaining({+ code: 'INVALID_ENVELOPE',+ certainty: 'definite',+ retryable: false,+ }),+ }),+ );+ expect(store.retry).not.toHaveBeenCalled();+ });++ it('dead-letters an exhausted row without dispatching again', async () => {+ const store = storeWith({+ claimNext: jest.fn(async () => delivery({ attempts: 8 })),+ });+ const dispatch = jest.fn(async () => successResult());+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START },+ { concurrency: 1, maxAttempts: 8 },+ );++ await engine.runTick();++ expect(dispatch).not.toHaveBeenCalled();+ expect(store.beginAttempt).not.toHaveBeenCalled();+ expect(store.dead).toHaveBeenCalledWith(expect.objectContaining({ claimToken: 'claim-1' }));+ });++ it('rechecks a leased predecessor promptly instead of waiting for its full lease', async () => {+ const store = storeWith({+ findEarlierUnsettled: jest.fn(async () => ({+ availableAt: new Date(START.getTime() + 2_000),+ leaseUntil: new Date(START.getTime() + 4_000),+ })),+ });+ const dispatch = jest.fn(async () => successResult());+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START },+ { concurrency: 1 },+ );++ await engine.runTick();++ expect(store.release).toHaveBeenCalledWith({+ id: 'delivery-row-1',+ workerId: expect.any(String),+ claimToken: 'claim-1',+ availableAt: new Date(START.getTime() + 250),+ });+ expect(store.beginAttempt).not.toHaveBeenCalled();+ expect(dispatch).not.toHaveBeenCalled();+ });++ it('does not poll ahead of a pending predecessor eligibility time', async () => {+ const store = storeWith({+ findEarlierUnsettled: jest.fn(async () => ({+ availableAt: new Date(START.getTime() + 4_000),+ })),+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: async () => successResult(), now: () => START },+ { concurrency: 1 },+ );++ await engine.runTick();++ expect(store.release).toHaveBeenCalledWith(+ expect.objectContaining({ availableAt: new Date(START.getTime() + 4_000) }),+ );+ });++ it('starts independent deliveries up to the configured concurrency', async () => {+ let releaseDispatch: (() => void) | undefined;+ const gate = new Promise<void>((resolve) => {+ releaseDispatch = resolve;+ });+ let dispatchStarts = 0;+ let bothStarted: (() => void) | undefined;+ const started = new Promise<void>((resolve) => {+ bothStarted = resolve;+ });+ const records = [delivery(), delivery({ id: 'delivery-row-2', claimToken: 'claim-2' })];+ const store = storeWith({+ claimNext: jest.fn(async () => records.shift() ?? null),+ });+ const dispatch = jest.fn(async () => {+ dispatchStarts += 1;+ if (dispatchStarts === 2) {+ bothStarted?.();+ }+ await gate;+ return successResult();+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START },+ { concurrency: 2 },+ );++ const tick = engine.runTick();+ await started;+ expect(dispatch).toHaveBeenCalledTimes(2);+ releaseDispatch?.();+ await tick;+ });++ it('coalesces overlapping ticks into one claim pass', async () => {+ let releaseClaim: (() => void) | undefined;+ const gate = new Promise<void>((resolve) => {+ releaseClaim = resolve;+ });+ const claimNext = jest.fn(async () => {+ await gate;+ return null;+ });+ const store = storeWith({ claimNext });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: async () => successResult() },+ { concurrency: 1 },+ );++ const first = engine.runTick();+ const second = engine.runTick();+ releaseClaim?.();+ await Promise.all([first, second]);+ expect(claimNext).toHaveBeenCalledTimes(1);+ });++ it('uses one claim when an idle worker has no due work', async () => {+ const claimNext = jest.fn(async () => null);+ const engine = createAgentTriggerDeliveryEngine(+ { store: storeWith({ claimNext }), dispatch: async () => successResult() },+ { concurrency: 4 },+ );++ await expect(engine.runTick()).resolves.toBe(0);++ expect(claimNext).toHaveBeenCalledTimes(1);+ });++ it('fills an open worker slot while an earlier independent delivery is still running', async () => {+ const records = [delivery({ envelope: { delivery: 1 } })];+ const store = storeWith({+ claimNext: jest.fn(async () => records.shift() ?? null),+ });+ let releaseFirst: (() => void) | undefined;+ const firstGate = new Promise<void>((resolve) => {+ releaseFirst = resolve;+ });+ let releaseSecond: (() => void) | undefined;+ const secondGate = new Promise<void>((resolve) => {+ releaseSecond = resolve;+ });+ let firstStarted: (() => void) | undefined;+ const firstStart = new Promise<void>((resolve) => {+ firstStarted = resolve;+ });+ let secondStarted: (() => void) | undefined;+ const secondStart = new Promise<void>((resolve) => {+ secondStarted = resolve;+ });+ const dispatch = jest.fn(async (value: unknown) => {+ const deliveryNumber = (value as { delivery: number }).delivery;+ if (deliveryNumber === 1) {+ firstStarted?.();+ await firstGate;+ } else {+ secondStarted?.();+ await secondGate;+ }+ return successResult();+ });+ const engine = createAgentTriggerDeliveryEngine({ store, dispatch }, { concurrency: 2 });++ engine.start();+ await firstStart;+ records.push(+ delivery({ id: 'delivery-row-2', claimToken: 'claim-2', envelope: { delivery: 2 } }),+ );+ engine.wake();+ await secondStart;++ expect(dispatch).toHaveBeenCalledTimes(2);+ releaseFirst?.();+ releaseSecond?.();+ await engine.stop();+ });++ it('retries ambiguously when execution succeeds but result persistence fails', async () => {+ const store = storeWith({+ complete: jest.fn(async () => Promise.reject(new Error('mongo unavailable'))),+ });+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch: async () => successResult(), now: () => START, random: () => 0 },+ { concurrency: 1, retryBaseMs: 1_000 },+ );++ await engine.runTick();++ expect(store.retry).toHaveBeenCalledWith(+ expect.objectContaining({+ claimToken: 'claim-1',+ availableAt: new Date(START.getTime() + 500),+ error: expect.objectContaining({+ code: 'RESULT_PERSISTENCE_FAILED',+ certainty: 'ambiguous',+ retryable: true,+ }),+ }),+ );+ });++ it('aborts in-flight dispatch during stop and persists a retry', async () => {+ const store = storeWith();+ let dispatchStarted: (() => void) | undefined;+ const started = new Promise<void>((resolve) => {+ dispatchStarted = resolve;+ });+ const dispatch = jest.fn(+ (+ _envelope: unknown,+ options?: { signal?: AbortSignal },+ ): Promise<AgentTriggerExecutionResult> =>+ new Promise((_, reject) => {+ dispatchStarted?.();+ options?.signal?.addEventListener(+ 'abort',+ () =>+ reject(+ new AgentTriggerExecutionError('stopping', {+ mode: 'fire',+ certainty: 'definite',+ retryable: true,+ code: 'ABORTED',+ }),+ ),+ { once: true },+ );+ }),+ );+ const engine = createAgentTriggerDeliveryEngine(+ { store, dispatch, now: () => START },+ { concurrency: 1 },+ );++ const tick = engine.runTick();+ await started;+ await engine.stop();+ await tick;++ expect(store.retry).toHaveBeenCalledWith(+ expect.objectContaining({+ claimToken: 'claim-1',+ error: expect.objectContaining({ code: 'ABORTED', retryable: true }),+ }),+ );+ });+});diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.tsnew file mode 100644index 0000000000..eb3e0a42a9--- /dev/null+++ b/packages/api/src/agents/triggers/engine.ts@@ -0,0 +1,590 @@+import { randomUUID } from 'node:crypto';+import { logger, runAsSystem } from '@librechat/data-schemas';+import type { AgentTriggerExecutionResult } from './host';+import { AgentTriggerDispatchError } from './dispatch';+import { AgentTriggerExecutionError } from './host';++const DEFAULT_CONCURRENCY = 4;+const DEFAULT_LEASE_MS = 2 * 60_000;+const DEFAULT_MAX_ATTEMPTS = 8;+const DEFAULT_RETRY_BASE_MS = 1_000;+const DEFAULT_RETRY_CAP_MS = 5 * 60_000;+const DEFAULT_TICK_MS = 1_000;+const ORDERING_RECHECK_MS = 250;+const DEFAULT_DEFER_MS = 5_000;+const MAX_RETRY_AFTER_MS = 24 * 60 * 60_000;++/** A pre-dispatch condition that must not consume the delivery's retry budget. */+export class AgentTriggerDeliveryDeferredError extends Error {+ readonly delayMs: number;++ constructor(message: string, delayMs: number = DEFAULT_DEFER_MS) {+ super(message);+ this.name = 'AgentTriggerDeliveryDeferredError';+ this.delayMs = positiveInteger(delayMs, DEFAULT_DEFER_MS, 'delayMs');+ }+}++export type AgentTriggerDeliveryStatus = 'staging' | 'pending' | 'leased' | 'succeeded' | 'dead';++export interface AgentTriggerDeliveryFailure {+ code: string;+ message: string;+ certainty: 'definite' | 'ambiguous';+ retryable: boolean;+ attemptedAt: Date;+ status?: number;+}++export interface AgentTriggerDeliveryRecord {+ id: string;+ user: string;+ claimToken: string;+ deliveryKey: string;+ fingerprint: string;+ orderingKey: string;+ laneSequence: number;+ envelope: unknown;+ status: AgentTriggerDeliveryStatus;+ attempts: number;+ availableAt: Date;+ createdAt: Date;+ leaseBy?: string;+ leaseUntil?: Date;+ lastError?: AgentTriggerDeliveryFailure;+}++export interface AgentTriggerOrderingBlock {+ availableAt: Date;+ leaseUntil?: Date;+}++export interface AgentTriggerDeliveryStore {+ claimNext: (input: {+ workerId: string;+ claimToken: string;+ now: Date;+ leaseUntil: Date;+ }) => Promise<AgentTriggerDeliveryRecord | null>;+ findEarlierUnsettled: (+ delivery: AgentTriggerDeliveryRecord,+ ) => Promise<AgentTriggerOrderingBlock | null>;+ release: (input: {+ id: string;+ workerId: string;+ claimToken: string;+ availableAt: Date;+ }) => Promise<boolean>;+ beginAttempt: (input: {+ id: string;+ workerId: string;+ claimToken: string;+ now: Date;+ }) => Promise<number | null>;+ defer: (input: {+ id: string;+ workerId: string;+ claimToken: string;+ attempt: number;+ availableAt: Date;+ }) => Promise<boolean>;+ complete: (input: {+ id: string;+ workerId: string;+ claimToken: string;+ attempt: number;+ result: AgentTriggerExecutionResult;+ settledAt: Date;+ }) => Promise<boolean>;+ retry: (input: {+ id: string;+ workerId: string;+ claimToken: string;+ attempt: number;+ error: AgentTriggerDeliveryFailure;+ availableAt: Date;+ }) => Promise<boolean>;+ dead: (input: {+ id: string;+ workerId: string;+ claimToken: string;+ attempt: number;+ error: AgentTriggerDeliveryFailure;+ settledAt: Date;+ }) => Promise<boolean>;+}++export interface AgentTriggerDeliveryEngineOptions {+ concurrency?: number;+ leaseMs?: number;+ maxAttempts?: number;+ retryBaseMs?: number;+ retryCapMs?: number;+ tickMs?: number;+}++export interface AgentTriggerDeliveryEngineDeps {+ store: AgentTriggerDeliveryStore;+ dispatch: (+ envelope: unknown,+ options?: { signal?: AbortSignal },+ ) => Promise<AgentTriggerExecutionResult>;+ now?: () => Date;+ random?: () => number;+ workerId?: string;+}++export interface AgentTriggerDeliveryEngine {+ start: () => void;+ stop: () => Promise<void>;+ cancelUser: (userId: string) => Promise<void>;+ releaseUserCancellation: (userId: string) => void;+ wake: () => void;+ runTick: () => Promise<number>;+}++function positiveInteger(value: number | undefined, fallback: number, name: string): number {+ const resolved = value ?? fallback;+ if (!Number.isSafeInteger(resolved) || resolved <= 0) {+ throw new TypeError(`${name} must be a positive integer`);+ }+ return resolved;+}++function failure(error: unknown, attemptedAt: Date): AgentTriggerDeliveryFailure {+ if (error instanceof AgentTriggerExecutionError) {+ return {+ code: error.code ?? 'DELIVERY_REJECTED',+ message: error.message,+ certainty: error.certainty,+ retryable: error.retryable,+ attemptedAt,+ ...(error.status != null && { status: error.status }),+ };+ }+ if (error instanceof AgentTriggerDispatchError) {+ return {+ code: 'INVALID_ENVELOPE',+ message: error.message,+ certainty: 'definite',+ retryable: false,+ attemptedAt,+ };+ }+ return {+ code: 'DELIVERY_FAILED',+ message: error instanceof Error ? error.message : String(error),+ certainty: 'definite',+ retryable: true,+ attemptedAt,+ };+}++function retryAt(+ error: unknown,+ attempt: number,+ now: Date,+ baseMs: number,+ capMs: number,+ random: () => number,+): Date {+ if (error instanceof AgentTriggerExecutionError && error.retryAfter != null) {+ const seconds = Number(error.retryAfter);+ if (Number.isFinite(seconds) && seconds >= 0) {+ return new Date(now.getTime() + Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS));+ }+ const absolute = Date.parse(error.retryAfter);+ if (Number.isFinite(absolute) && absolute > now.getTime()) {+ return new Date(Math.min(absolute, now.getTime() + MAX_RETRY_AFTER_MS));+ }+ }+ const exponent = Math.min(attempt - 1, 30);+ const delay = Math.min(baseMs * 2 ** exponent, capMs);+ return new Date(now.getTime() + Math.floor(delay / 2 + random() * (delay / 2)));+}++function isAccountDeletionDeferral(error: unknown): boolean {+ return (+ error instanceof AgentTriggerExecutionError &&+ error.code === 'ACCOUNT_DELETION_IN_PROGRESS' &&+ (error.status === 401 || error.status === 409)+ );+}++/** Durable, lease-fenced delivery runner shared by every trusted event source. */+export function createAgentTriggerDeliveryEngine(+ deps: AgentTriggerDeliveryEngineDeps,+ options: AgentTriggerDeliveryEngineOptions = {},+): AgentTriggerDeliveryEngine {+ const concurrency = positiveInteger(options.concurrency, DEFAULT_CONCURRENCY, 'concurrency');+ const leaseMs = positiveInteger(options.leaseMs, DEFAULT_LEASE_MS, 'leaseMs');+ const maxAttempts = positiveInteger(options.maxAttempts, DEFAULT_MAX_ATTEMPTS, 'maxAttempts');+ const retryBaseMs = positiveInteger(options.retryBaseMs, DEFAULT_RETRY_BASE_MS, 'retryBaseMs');+ const retryCapMs = positiveInteger(options.retryCapMs, DEFAULT_RETRY_CAP_MS, 'retryCapMs');+ const tickMs = positiveInteger(options.tickMs, DEFAULT_TICK_MS, 'tickMs');+ const now = deps.now ?? (() => new Date());+ const random = deps.random ?? Math.random;+ const workerId = deps.workerId ?? `${process.pid}-${randomUUID()}`;+ const controllers = new Map<AbortController, string>();+ const processing = new Set<Promise<void>>();+ const processingByUser = new Map<string, Set<Promise<void>>>();+ const cancelledUsers = new Set<string>();+ let stopped = false;+ let started = false;+ let repumpRequested = false;+ let timer: NodeJS.Timeout | undefined;+ let activeClaim: Promise<{ count: number; processing: Promise<void>[] }> | undefined;++ const processDelivery = async (delivery: AgentTriggerDeliveryRecord): Promise<void> => {+ const userId = String(delivery.user);+ if (cancelledUsers.has(userId)) {+ await deps.store.release({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ availableAt: now(),+ });+ return;+ }++ const block = await deps.store.findEarlierUnsettled(delivery);+ if (block != null) {+ const recheckAt = now().getTime() + ORDERING_RECHECK_MS;+ const nextCheck =+ block.leaseUntil == null ? Math.max(recheckAt, block.availableAt.getTime()) : recheckAt;+ await deps.store.release({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ availableAt: new Date(nextCheck),+ });+ return;+ }++ if (delivery.attempts >= maxAttempts) {+ const recorded =+ delivery.lastError ??+ failure(new Error('Delivery attempt limit was already exhausted'), now());+ const deadLettered = await deps.store.dead({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ attempt: delivery.attempts,+ error: recorded,+ settledAt: now(),+ });+ if (deadLettered) {+ logger.error('[agent-triggers] delivery dead-lettered after exhausting retries', {+ deliveryKey: delivery.deliveryKey,+ attempts: delivery.attempts,+ code: recorded.code,+ });+ }+ return;+ }++ if (stopped) {+ await deps.store.release({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ availableAt: now(),+ });+ return;+ }++ if (cancelledUsers.has(userId)) {+ await deps.store.release({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ availableAt: now(),+ });+ return;+ }++ const attempt = await deps.store.beginAttempt({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ now: now(),+ });+ if (attempt == null) {+ return;+ }++ const controller = new AbortController();+ controllers.set(controller, userId);+ if (stopped || cancelledUsers.has(userId)) {+ controller.abort(new Error('Agent trigger delivery engine is stopping'));+ }+ try {+ let result: AgentTriggerExecutionResult;+ try {+ result = await deps.dispatch(delivery.envelope, { signal: controller.signal });+ } catch (error) {+ const attemptedAt = now();+ const deletionCancelled = controller.signal.aborted && cancelledUsers.has(userId);+ const deletionRejected = isAccountDeletionDeferral(error);+ if (+ error instanceof AgentTriggerDeliveryDeferredError ||+ deletionCancelled ||+ deletionRejected+ ) {+ const delayMs =+ error instanceof AgentTriggerDeliveryDeferredError ? error.delayMs : DEFAULT_DEFER_MS;+ const availableAt = new Date(attemptedAt.getTime() + delayMs);+ const deferred = await deps.store.defer({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ attempt,+ availableAt,+ });+ if (deferred) {+ logger.info('[agent-triggers] delivery deferred without consuming an attempt', {+ deliveryKey: delivery.deliveryKey,+ reason: deletionCancelled || deletionRejected ? 'account_deletion' : 'pre_dispatch',+ availableAt: availableAt.toISOString(),+ });+ }+ return;+ }+ const recorded = failure(error, attemptedAt);+ if (!recorded.retryable || attempt >= maxAttempts) {+ const deadLettered = await deps.store.dead({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ attempt,+ error: recorded,+ settledAt: attemptedAt,+ });+ if (deadLettered) {+ logger.error('[agent-triggers] delivery dead-lettered', {+ deliveryKey: delivery.deliveryKey,+ attempt,+ code: recorded.code,+ certainty: recorded.certainty,+ });+ }+ return;+ }+ const availableAt = retryAt(error, attempt, attemptedAt, retryBaseMs, retryCapMs, random);+ const retrying = await deps.store.retry({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ attempt,+ error: recorded,+ availableAt,+ });+ if (retrying) {+ logger.warn('[agent-triggers] delivery scheduled for retry', {+ deliveryKey: delivery.deliveryKey,+ attempt,+ code: recorded.code,+ certainty: recorded.certainty,+ availableAt: availableAt.toISOString(),+ });+ }+ return;+ }++ const settledAt = now();+ try {+ await deps.store.complete({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ attempt,+ result,+ settledAt,+ });+ } catch (error) {+ const recorded: AgentTriggerDeliveryFailure = {+ code: 'RESULT_PERSISTENCE_FAILED',+ message: error instanceof Error ? error.message : String(error),+ certainty: 'ambiguous',+ retryable: true,+ attemptedAt: settledAt,+ };+ const availableAt = retryAt(error, attempt, settledAt, retryBaseMs, retryCapMs, random);+ const retrying = await deps.store.retry({+ id: delivery.id,+ workerId,+ claimToken: delivery.claimToken,+ attempt,+ error: recorded,+ availableAt,+ });+ if (retrying) {+ logger.warn(+ '[agent-triggers] accepted delivery result could not be persisted; retrying',+ {+ deliveryKey: delivery.deliveryKey,+ attempt,+ availableAt: availableAt.toISOString(),+ },+ );+ }+ }+ } finally {+ controllers.delete(controller);+ }+ };++ const claimOne = (): Promise<AgentTriggerDeliveryRecord | null> => {+ const claimedAt = now();+ return deps.store.claimNext({+ workerId,+ claimToken: randomUUID(),+ now: claimedAt,+ leaseUntil: new Date(claimedAt.getTime() + leaseMs),+ });+ };++ const runClaimPass = async (): Promise<{ count: number; processing: Promise<void>[] }> => {+ if (stopped) {+ return { count: 0, processing: [] };+ }+ const openSlots = concurrency - processing.size;+ if (openSlots <= 0) {+ return { count: 0, processing: [] };+ }++ const deliveries: AgentTriggerDeliveryRecord[] = [];+ try {+ const first = await claimOne();+ if (first == null) {+ return { count: 0, processing: [] };+ }+ deliveries.push(first);+ } catch (error) {+ logger.error('[agent-triggers] delivery claim failed:', error);+ return { count: 0, processing: [] };+ }++ if (openSlots > 1) {+ const claimed = await Promise.allSettled(+ Array.from({ length: openSlots - 1 }, () => claimOne()),+ );+ for (const result of claimed) {+ if (result.status === 'fulfilled' && result.value != null) {+ deliveries.push(result.value);+ } else if (result.status === 'rejected') {+ logger.error('[agent-triggers] delivery claim failed:', result.reason);+ }+ }+ }++ const batch: Promise<void>[] = [];+ for (const delivery of deliveries) {+ const userId = String(delivery.user);+ const task = runAsSystem(() => processDelivery(delivery)).catch((error) => {+ logger.error('[agent-triggers] delivery processing failed:', error);+ });+ const tracked = task.finally(() => {+ processing.delete(tracked);+ const userProcessing = processingByUser.get(userId);+ userProcessing?.delete(tracked);+ if (userProcessing?.size === 0) {+ processingByUser.delete(userId);+ }+ if (started && !stopped) {+ queueMicrotask(wake);+ }+ });+ processing.add(tracked);+ const userProcessing = processingByUser.get(userId) ?? new Set<Promise<void>>();+ userProcessing.add(tracked);+ processingByUser.set(userId, userProcessing);+ batch.push(tracked);+ }+ return { count: deliveries.length, processing: batch };+ };++ const claimAvailable = (): Promise<{ count: number; processing: Promise<void>[] }> => {+ if (activeClaim != null) {+ return activeClaim;+ }+ activeClaim = runAsSystem(runClaimPass).finally(() => {+ activeClaim = undefined;+ if (repumpRequested && !stopped) {+ repumpRequested = false;+ queueMicrotask(wake);+ }+ });+ return activeClaim;+ };++ const runTick = async (): Promise<number> => {+ const batch = await claimAvailable();+ await Promise.allSettled(batch.processing);+ return batch.count;+ };++ function wake(): void {+ if (stopped) {+ return;+ }+ if (activeClaim != null) {+ repumpRequested = true;+ return;+ }+ void claimAvailable().catch((error) =>+ logger.error('[agent-triggers] delivery claim pass failed:', error),+ );+ }++ const schedule = () => {+ if (stopped) {+ return;+ }+ timer = setTimeout(() => {+ wake();+ schedule();+ }, tickMs);+ timer.unref();+ };++ return {+ start: () => {+ if (started || stopped) {+ return;+ }+ started = true;+ wake();+ schedule();+ },+ stop: async () => {+ stopped = true;+ if (timer != null) {+ clearTimeout(timer);+ }+ for (const controller of controllers.keys()) {+ controller.abort();+ }+ await activeClaim?.catch(() => undefined);+ await Promise.allSettled([...processing]);+ },+ cancelUser: async (userId) => {+ cancelledUsers.add(userId);+ for (const [controller, activeUserId] of controllers) {+ if (activeUserId === userId) {+ controller.abort(new Error('Agent trigger delivery cancelled for account deletion'));+ }+ }+ await activeClaim?.catch(() => undefined);+ const userProcessing = processingByUser.get(userId);+ if (userProcessing != null) {+ await Promise.allSettled([...userProcessing]);+ }+ },+ releaseUserCancellation: (userId) => {+ cancelledUsers.delete(userId);+ },+ wake,+ runTick,+ };+}diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.tsindex d94a935b05..86f4ca6f70 100644--- a/packages/api/src/agents/triggers/host.spec.ts+++ b/packages/api/src/agents/triggers/host.spec.ts@@ -120,6 +120,7 @@ describe('createAgentTriggerExecutionHost fire adapter', () => {expect(String(input)).toBe('https://chat.example.test/base/api/agents/chat/agents');const headers = new Headers(init?.headers);expect(headers.get('authorization')).toBe('Bearer signed-token');+ expect(headers.get('x-lc-agent-trigger')).toBe('1');expect(headers.get('x-request-id')).toBe(idempotencyKey);expect(headers.get('x-librechat-generation-protocol')).toBe('2');expect(headers.get('user-agent')).toContain('LibreChat-Agent-Trigger/1');@@ -499,6 +500,7 @@ describe('createAgentTriggerExecutionHost steer adapter', () => {expect(String(input)).toBe('https://chat.example.test/base/api/agents/chat/steer/deliver');const headers = new Headers(init?.headers);expect(headers.get('authorization')).toBe('Bearer signed-token');+ expect(headers.get('x-lc-agent-trigger')).toBe('1');expect(headers.get('x-request-id')).toBe(idempotencyKey);expect(headers.get('x-librechat-generation-protocol')).toBe('2');expect(JSON.parse(String(init?.body))).toEqual({diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.tsindex a59be869b2..730a75c10d 100644--- a/packages/api/src/agents/triggers/host.ts+++ b/packages/api/src/agents/triggers/host.ts@@ -467,6 +467,7 @@ async function fire(Authorization: `Bearer ${token}`,'Content-Type': 'application/json','User-Agent': TRIGGER_USER_AGENT,+ 'x-lc-agent-trigger': '1','x-request-id': context.idempotencyKey,[GENERATION_PROTOCOL_HEADER]: '2',},@@ -650,6 +651,7 @@ async function steer(Authorization: `Bearer ${token}`,'Content-Type': 'application/json','User-Agent': TRIGGER_USER_AGENT,+ 'x-lc-agent-trigger': '1','x-request-id': context.idempotencyKey,[GENERATION_PROTOCOL_HEADER]: '2',},diff --git a/packages/api/src/agents/triggers/index.ts b/packages/api/src/agents/triggers/index.tsindex 3c75f2ba09..ab2c692942 100644--- a/packages/api/src/agents/triggers/index.ts+++ b/packages/api/src/agents/triggers/index.ts@@ -1,3 +1,6 @@export * from './dispatch';+export * from './delivery';export * from './envelope';export * from './host';+export * from './service';+export * from './engine';diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsnew file mode 100644index 0000000000..f9e0f7a329--- /dev/null+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -0,0 +1,332 @@+import { getTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas';+import type { AgentTriggerDeliveryPersistence, AgentTriggerStoredRecord } from './service';+import { AgentTriggerServiceUnavailableError, createAgentTriggerService } from './service';+import { __resetShutdownStateForTests } from '../../app/shutdown';+import { createAgentTriggerEnvelope } from './envelope';++jest.mock('@librechat/data-schemas', () => {+ const actual = jest.requireActual('@librechat/data-schemas');+ return {+ ...actual,+ logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+});++const START = new Date('2026-08-17T12:00:00.000Z');++const envelope = () =>+ createAgentTriggerEnvelope({+ mode: 'fire',+ requestId: 'request-1',+ deliveryId: 'delivery-1',+ receivedAt: 20,+ principal: { id: '507f1f77bcf86cd799439011', tenantId: 'tenant-1' },+ target: { agentId: 'agent-1' },+ event: {+ id: 'event-1',+ type: 'resource.ready',+ occurredAt: 10,+ source: { id: 'source-1', type: 'webhook' },+ },+ input: 'Handle the ready resource.',+ });++function deliveryRecord(overrides: Partial<AgentTriggerStoredRecord> = {}) {+ return {+ id: 'delivery-row-1',+ user: '507f1f77bcf86cd799439011',+ deliveryKey: 'trigger_1',+ fingerprint: 'fingerprint-1',+ orderingKey: 'ordering-1',+ laneSequence: 1,+ envelope: envelope(),+ tenantId: 'tenant-1',+ status: 'pending' as const,+ attempts: 0,+ availableAt: START,+ createdAt: START,+ ...overrides,+ };+}++function deliveryMethods(+ overrides: Partial<AgentTriggerDeliveryPersistence> = {},+): AgentTriggerDeliveryPersistence {+ return {+ ensureAgentTriggerDeliveryIndexes: jest.fn(async () => undefined),+ enqueueAgentTriggerDelivery: jest.fn(async (input) => ({+ delivery: deliveryRecord({+ deliveryKey: input.deliveryKey,+ fingerprint: input.fingerprint,+ orderingKey: input.orderingKey,+ laneSequence: 1,+ envelope: input.envelope,+ availableAt: input.availableAt,+ }),+ replayed: false,+ })),+ claimNextAgentTriggerDelivery: jest.fn(async () => null),+ findEarlierAgentTriggerDelivery: jest.fn(async () => null),+ releaseAgentTriggerDelivery: jest.fn(async () => true),+ beginAgentTriggerDeliveryAttempt: jest.fn(async () => 1),+ deferAgentTriggerDeliveryAttempt: jest.fn(async () => true),+ completeAgentTriggerDelivery: jest.fn(async () => true),+ retryAgentTriggerDelivery: jest.fn(async () => true),+ deadLetterAgentTriggerDelivery: jest.fn(async () => true),+ getAgentTriggerDelivery: jest.fn(async () => null),+ getAgentTriggerDeadLetters: jest.fn(async () => []),+ requeueAgentTriggerDelivery: jest.fn(async () => null),+ countActiveAgentTriggerDeliveriesByUser: jest.fn(async () => 0),+ recoverAgentTriggerLanePublications: jest.fn(async () => 0),+ reclaimInactiveAgentTriggerLanes: jest.fn(async () => 0),+ prepareAgentTriggerUserPurge: jest.fn(async () => undefined),+ cancelAgentTriggerUserPurge: jest.fn(async () => true),+ recoverAgentTriggerUserPurges: jest.fn(async () => 0),+ deleteAgentTriggerDeliveriesByUser: jest.fn(async () => undefined),+ ...overrides,+ };+}++describe('durable agent trigger service', () => {+ const originalSelfUrl = process.env.AGENT_TRIGGERS_SELF_URL;++ beforeEach(() => {+ __resetShutdownStateForTests();+ delete process.env.AGENT_TRIGGERS_SELF_URL;+ });++ afterEach(() => {+ __resetShutdownStateForTests();+ });++ afterAll(() => {+ if (originalSelfUrl == null) {+ delete process.env.AGENT_TRIGGERS_SELF_URL;+ } else {+ process.env.AGENT_TRIGGERS_SELF_URL = originalSelfUrl;+ }+ });++ it('fails closed before indexes and the delivery engine are ready', async () => {+ const service = createAgentTriggerService({ methods: deliveryMethods() });++ await expect(service.enqueue(envelope())).rejects.toBeInstanceOf(+ AgentTriggerServiceUnavailableError,+ );+ await service.stop();+ });++ it('refuses to arm without a reachable self origin', async () => {+ const service = createAgentTriggerService({ methods: deliveryMethods() });++ await expect(service.initialize()).rejects.toThrow(+ 'requires a valid listener address or AGENT_TRIGGERS_SELF_URL',+ );+ await service.stop();+ });++ it('initializes indexes, enqueues normalized source-neutral work, and starts claims', async () => {+ const methods = deliveryMethods();+ const service = createAgentTriggerService({+ methods,+ mintToken: () => 'token',+ fetch: async () => new Response('{}', { status: 500 }),+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });++ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });+ const receipt = await service.enqueue(envelope(), {+ orderingKey: 'match-42',+ availableAt: START,+ });++ expect(methods.ensureAgentTriggerDeliveryIndexes).toHaveBeenCalledTimes(1);+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();+ expect(methods.enqueueAgentTriggerDelivery).toHaveBeenCalledWith(+ expect.objectContaining({+ deliveryKey: expect.stringMatching(/^trigger_/),+ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/),+ orderingKey: expect.stringMatching(/^trigger_lane_/),+ user: '507f1f77bcf86cd799439011',+ tenantId: 'tenant-1',+ availableAt: START,+ }),+ );+ expect(receipt).toMatchObject({+ id: 'delivery-row-1',+ status: 'pending',+ replayed: false,+ availableAt: START,+ });+ await service.stop();+ });++ it('rejects enqueue before persistence when the principal was deleted', async () => {+ const methods = deliveryMethods();+ const service = createAgentTriggerService({+ methods,+ isPrincipalActive: async () => false,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });++ await expect(service.enqueue(envelope())).rejects.toThrow(+ 'Agent trigger delivery principal is no longer active',+ );+ expect(methods.enqueueAgentTriggerDelivery).not.toHaveBeenCalled();+ await service.stop();+ });++ it('preserves a just-enqueued row when deletion wins the admission race', async () => {+ const methods = deliveryMethods();+ const isPrincipalActive = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false);+ const service = createAgentTriggerService({+ methods,+ isPrincipalActive,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });++ await expect(service.enqueue(envelope())).rejects.toThrow(+ 'Agent trigger delivery principal is no longer active',+ );+ expect(methods.countActiveAgentTriggerDeliveriesByUser).toHaveBeenCalled();+ expect(methods.deleteAgentTriggerDeliveriesByUser).not.toHaveBeenCalled();+ await service.stop();+ });++ it("drains without data loss, then purges one user's deliveries after commit", async () => {+ const countActiveAgentTriggerDeliveriesByUser = jest+ .fn()+ .mockResolvedValueOnce(1)+ .mockResolvedValueOnce(0);+ const deleteAgentTriggerDeliveriesByUser = jest.fn(async () => {+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);+ });+ const methods = deliveryMethods({+ countActiveAgentTriggerDeliveriesByUser,+ deleteAgentTriggerDeliveriesByUser,+ });+ const service = createAgentTriggerService({+ methods,+ userDrainPollMs: 1,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });++ await service.drainUser('507f1f77bcf86cd799439011');++ expect(countActiveAgentTriggerDeliveriesByUser).toHaveBeenCalledTimes(2);+ expect(deleteAgentTriggerDeliveriesByUser).not.toHaveBeenCalled();++ await service.stop();+ await service.purgeUser('507f1f77bcf86cd799439011');++ expect(deleteAgentTriggerDeliveriesByUser).toHaveBeenCalledWith('507f1f77bcf86cd799439011');+ });++ it('arms and disarms post-commit purge recovery in system context', async () => {+ const prepareAgentTriggerUserPurge = jest.fn(async () => {+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);+ });+ const cancelAgentTriggerUserPurge = jest.fn(async () => {+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);+ return true;+ });+ const methods = deliveryMethods({+ prepareAgentTriggerUserPurge,+ cancelAgentTriggerUserPurge,+ });+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });+ const fence = new Date(START);++ await service.prepareUserPurge('507f1f77bcf86cd799439011', fence, 'tenant-1');+ await expect(service.cancelUserPurge('507f1f77bcf86cd799439011', fence)).resolves.toBe(true);++ expect(prepareAgentTriggerUserPurge).toHaveBeenCalledWith(+ '507f1f77bcf86cd799439011',+ fence,+ 'tenant-1',+ );+ await service.stop();+ });++ it('retries recoverable post-commit purges after a transient sweep failure', async () => {+ const recoverAgentTriggerUserPurges = jest+ .fn()+ .mockRejectedValueOnce(new Error('mongo unavailable'))+ .mockResolvedValue(1);+ const service = createAgentTriggerService({+ methods: deliveryMethods({ recoverAgentTriggerUserPurges }),+ purgeRecoveryIntervalMs: 5,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });++ await new Promise((resolve) => setTimeout(resolve, 20));++ expect(recoverAgentTriggerUserPurges.mock.calls.length).toBeGreaterThanOrEqual(2);+ await service.stop();+ });++ it('keeps trusted operational reads and requeue in system tenant context', async () => {+ const getAgentTriggerDeadLetters = jest.fn(async () => {+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);+ return [deliveryRecord({ status: 'dead' })];+ });+ const requeueAgentTriggerDelivery = jest.fn(async () => {+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);+ return deliveryRecord();+ });+ const methods = deliveryMethods({+ getAgentTriggerDeadLetters,+ requeueAgentTriggerDelivery,+ });+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });++ await expect(service.getDeadLetters(10)).resolves.toHaveLength(1);+ await expect(service.requeue('delivery-row-1', START)).resolves.toMatchObject({+ status: 'pending',+ });+ await service.stop();+ });++ it('does not become ready when the required unique indexes fail', async () => {+ const methods = deliveryMethods({+ ensureAgentTriggerDeliveryIndexes: jest.fn(async () =>+ Promise.reject(new Error('index unavailable')),+ ),+ });+ const service = createAgentTriggerService({ methods });++ await expect(+ service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ }),+ ).rejects.toThrow('index unavailable');+ await expect(service.enqueue(envelope())).rejects.toBeInstanceOf(+ AgentTriggerServiceUnavailableError,+ );+ await service.stop();+ });+});diff --git a/packages/api/src/agents/triggers/service.spec.ts b/packages/api/src/agents/triggers/service.spec.tsnew file mode 100644index 0000000000..51fe993731--- /dev/null+++ b/packages/api/src/agents/triggers/service.spec.ts@@ -0,0 +1,126 @@+import jwt from 'jsonwebtoken';+import type { AgentTriggerFetch } from './host';+import { createAgentTriggerEnvelope } from './envelope';+import { AGENT_TRIGGER_SCOPE } from '../../crypto/jwt';+import { createAgentTriggerService } from './service';+import { AgentTriggerExecutionError } from './host';++const envelope = () =>+ createAgentTriggerEnvelope({+ mode: 'fire',+ requestId: 'request-1',+ deliveryId: 'delivery-1',+ receivedAt: 20,+ principal: { id: 'user-1', role: 'member', tenantId: 'tenant-1' },+ target: { agentId: 'agent-1' },+ event: {+ id: 'event-1',+ type: 'resource.ready',+ occurredAt: 10,+ source: { id: 'source-1', type: 'webhook' },+ },+ input: 'Handle the ready resource.',+ });++const accepted = () =>+ new Response(+ JSON.stringify({+ status: 'started',+ streamId: 'conversation-1',+ conversationId: 'conversation-1',+ generationCreatedAt: 25,+ }),+ { status: 200, headers: { 'content-type': 'application/json' } },+ );++describe('createAgentTriggerService', () => {+ const originalSecret = process.env.JWT_SECRET;+ const originalSelfUrl = process.env.AGENT_TRIGGERS_SELF_URL;++ beforeEach(() => {+ process.env.JWT_SECRET = 'trigger-test-secret';+ delete process.env.AGENT_TRIGGERS_SELF_URL;+ });++ afterAll(() => {+ process.env.JWT_SECRET = originalSecret;+ if (originalSelfUrl == null) {+ delete process.env.AGENT_TRIGGERS_SELF_URL;+ } else {+ process.env.AGENT_TRIGGERS_SELF_URL = originalSelfUrl;+ }+ });++ it('targets the bound listener and mints a scoped user token', async () => {+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(+ async () => accepted(),+ );+ const service = createAgentTriggerService({ fetch: fetcher });+ await service.initialize({ address: { address: '::', family: 'IPv6', port: 4123 } });++ await expect(service.dispatch(envelope())).resolves.toMatchObject({+ mode: 'fire',+ status: 'started',+ conversationId: 'conversation-1',+ });++ const [url, init] = fetcher.mock.calls[0];+ expect(String(url)).toBe('http://[::1]:4123/api/agents/chat/agents');+ const headers = new Headers(init?.headers);+ expect(headers.get('x-lc-agent-trigger')).toBe('1');+ const authorization = headers.get('authorization');+ expect(authorization).toMatch(/^Bearer /);+ expect(+ jwt.verify(authorization!.slice('Bearer '.length), process.env.JWT_SECRET!, {+ algorithms: ['HS256'],+ }),+ ).toMatchObject({ id: 'user-1', scope: AGENT_TRIGGER_SCOPE });+ });++ it('honors an operator self-URL override before listener initialization', async () => {+ process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal/base';+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(+ async () => accepted(),+ );+ const service = createAgentTriggerService({ fetch: fetcher, mintToken: () => 'token' });++ await service.dispatch(envelope());++ expect(String(fetcher.mock.calls[0][0])).toBe(+ 'https://triggers.internal/base/api/agents/chat/agents',+ );+ });++ it('rechecks the durable principal before dispatching queued or direct work', async () => {+ process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal';+ const fetcher = jest.fn(async () => accepted());+ const service = createAgentTriggerService({+ fetch: fetcher,+ mintToken: () => 'token',+ isPrincipalActive: async () => false,+ });++ await expect(service.dispatch(envelope())).rejects.toThrow(+ 'Agent trigger delivery principal is no longer active',+ );+ expect(fetcher).not.toHaveBeenCalled();+ });++ it('fails safely when dispatch starts before a listener or override exists', async () => {+ expect.hasAssertions();+ const service = createAgentTriggerService({+ fetch: jest.fn(async () => accepted()),+ mintToken: () => 'token',+ });++ await service.dispatch(envelope()).catch((error: unknown) => {+ expect(error).toBeInstanceOf(AgentTriggerExecutionError);+ expect(error).toMatchObject({+ mode: 'fire',+ certainty: 'definite',+ retryable: true,+ code: 'SETUP_FAILED',+ });+ });+ });+});diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsnew file mode 100644index 0000000000..ad8e155821--- /dev/null+++ b/packages/api/src/agents/triggers/service.ts@@ -0,0 +1,414 @@+import { logger, runAsSystem } from '@librechat/data-schemas';+import type {+ AgentTriggerDeliveryFailure,+ AgentTriggerDeliveryEngine,+ AgentTriggerDeliveryEngineOptions,+ AgentTriggerDeliveryRecord,+ AgentTriggerDeliveryStore,+} from './engine';+import type {+ AgentTriggerExecutionHost,+ AgentTriggerExecutionHostDeps,+ AgentTriggerExecutionResult,+} from './host';+import type { AgentTriggerEnqueueOptions, PreparedAgentTriggerDelivery } from './delivery';+import type { BoundAddress } from '../../app/origin';+import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine';+import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown';+import { generateAgentTriggerToken } from '../../crypto/jwt';+import { selfOriginFromAddress } from '../../app/origin';+import { prepareAgentTriggerDelivery } from './delivery';+import { createAgentTriggerExecutionHost } from './host';+import { parseAgentTriggerEnvelope } from './envelope';++export const AGENT_TRIGGER_TOKEN_TTL = '60s';+const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;+const DEFAULT_USER_DRAIN_POLL_MS = 100;+const DEFAULT_PURGE_RECOVERY_INTERVAL_MS = 30_000;+const DEFAULT_PURGE_RECOVERY_LIMIT = 25;++export interface AgentTriggerServiceOptions {+ address?: BoundAddress | string | null;+}++export interface AgentTriggerServiceDeps {+ fetch?: AgentTriggerExecutionHostDeps['fetch'];+ getTimezone?: AgentTriggerExecutionHostDeps['getTimezone'];+ mintToken?: AgentTriggerExecutionHostDeps['mintToken'];+ timeoutMs?: number;+ methods?: AgentTriggerDeliveryPersistence;+ deliveryOptions?: AgentTriggerDeliveryEngineOptions;+ isPrincipalActive?: (userId: string) => boolean | Promise<boolean>;+ userDrainTimeoutMs?: number;+ userDrainPollMs?: number;+ purgeRecoveryIntervalMs?: number;+ purgeRecoveryLimit?: number;+}++export interface AgentTriggerDeliveryReceipt {+ id: string;+ deliveryKey: string;+ status: AgentTriggerStoredRecord['status'];+ availableAt: Date;+ replayed: boolean;+}++export class AgentTriggerServiceUnavailableError extends AgentTriggerDeliveryDeferredError {+ constructor(message: string) {+ super(message);+ this.name = 'AgentTriggerServiceUnavailableError';+ }+}++export interface AgentTriggerStoredHistoryEntry {+ attempt: number;+ outcome: 'succeeded' | 'retry' | 'dead';+ at: Date;+ workerId: string;+ error?: AgentTriggerDeliveryFailure;+}++export interface AgentTriggerStoredRecord extends Omit<AgentTriggerDeliveryRecord, 'claimToken'> {+ claimToken?: string;+ tenantId?: string;+ result?: unknown;+ history?: AgentTriggerStoredHistoryEntry[];+ settledAt?: Date;+ expiresAt?: Date;+ requeueCount?: number;+}++/** Persistence contract implemented by the data-schemas method bundle. */+export interface AgentTriggerDeliveryPersistence {+ ensureAgentTriggerDeliveryIndexes: () => Promise<void>;+ enqueueAgentTriggerDelivery: (+ input: PreparedAgentTriggerDelivery,+ ) => Promise<{ delivery: AgentTriggerStoredRecord; replayed: boolean }>;+ claimNextAgentTriggerDelivery: AgentTriggerDeliveryStore['claimNext'];+ findEarlierAgentTriggerDelivery: AgentTriggerDeliveryStore['findEarlierUnsettled'];+ releaseAgentTriggerDelivery: AgentTriggerDeliveryStore['release'];+ beginAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['beginAttempt'];+ deferAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['defer'];+ completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];+ retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];+ deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];+ getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;+ getAgentTriggerDeadLetters: (limit?: number) => Promise<AgentTriggerStoredRecord[]>;+ requeueAgentTriggerDelivery: (+ id: string,+ availableAt: Date,+ ) => Promise<AgentTriggerStoredRecord | null>;+ countActiveAgentTriggerDeliveriesByUser: (userId: string, now: Date) => Promise<number>;+ recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;+ reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;+ prepareAgentTriggerUserPurge: (+ userId: string,+ fenceStartedAt: Date,+ tenantId?: string,+ ) => Promise<void>;+ cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;+ recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;+ deleteAgentTriggerDeliveriesByUser: (userId: string) => Promise<void>;+}++export interface AgentTriggerService {+ initialize: (options?: AgentTriggerServiceOptions) => Promise<void>;+ stop: () => Promise<void>;+ dispatch: (+ envelope: unknown,+ options?: { signal?: AbortSignal },+ ) => Promise<AgentTriggerExecutionResult>;+ enqueue: (+ envelope: unknown,+ options?: AgentTriggerEnqueueOptions,+ ) => Promise<AgentTriggerDeliveryReceipt>;+ getDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;+ getDeadLetters: (limit?: number) => Promise<AgentTriggerStoredRecord[]>;+ requeue: (id: string, availableAt?: Date) => Promise<AgentTriggerStoredRecord | null>;+ drainUser: (userId: string) => Promise<void>;+ prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;+ cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;+ purgeUser: (userId: string) => Promise<void>;+}++function createDeliveryStore(methods: AgentTriggerDeliveryPersistence): AgentTriggerDeliveryStore {+ return {+ claimNext: methods.claimNextAgentTriggerDelivery,+ findEarlierUnsettled: methods.findEarlierAgentTriggerDelivery,+ release: methods.releaseAgentTriggerDelivery,+ beginAttempt: methods.beginAgentTriggerDeliveryAttempt,+ defer: methods.deferAgentTriggerDeliveryAttempt,+ complete: methods.completeAgentTriggerDelivery,+ retry: methods.retryAgentTriggerDelivery,+ dead: methods.deadLetterAgentTriggerDelivery,+ };+}++function requireDeliveryOrigin(boundOrigin: string | undefined): void {+ const value = process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin;+ let url: URL;+ try {+ url = new URL(value ?? '');+ } catch {+ throw new AgentTriggerServiceUnavailableError(+ 'Durable agent trigger delivery requires a valid listener address or AGENT_TRIGGERS_SELF_URL',+ );+ }+ if ((url.protocol !== 'http:' && url.protocol !== 'https:') || url.username || url.password) {+ throw new AgentTriggerServiceUnavailableError(+ 'Durable agent trigger delivery requires an HTTP(S) self URL without credentials',+ );+ }+}++/** Production composition for trusted, in-process trigger producers. */+export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): AgentTriggerService {+ const userDrainTimeoutMs = deps.userDrainTimeoutMs ?? DEFAULT_USER_DRAIN_TIMEOUT_MS;+ const userDrainPollMs = deps.userDrainPollMs ?? DEFAULT_USER_DRAIN_POLL_MS;+ const purgeRecoveryIntervalMs =+ deps.purgeRecoveryIntervalMs ?? DEFAULT_PURGE_RECOVERY_INTERVAL_MS;+ const purgeRecoveryLimit = deps.purgeRecoveryLimit ?? DEFAULT_PURGE_RECOVERY_LIMIT;+ if (!Number.isSafeInteger(userDrainTimeoutMs) || userDrainTimeoutMs <= 0) {+ throw new TypeError('userDrainTimeoutMs must be a positive integer');+ }+ if (!Number.isSafeInteger(userDrainPollMs) || userDrainPollMs <= 0) {+ throw new TypeError('userDrainPollMs must be a positive integer');+ }+ if (!Number.isSafeInteger(purgeRecoveryIntervalMs) || purgeRecoveryIntervalMs <= 0) {+ throw new TypeError('purgeRecoveryIntervalMs must be a positive integer');+ }+ if (!Number.isSafeInteger(purgeRecoveryLimit) || purgeRecoveryLimit <= 0) {+ throw new TypeError('purgeRecoveryLimit must be a positive integer');+ }+ let boundOrigin: string | undefined;+ let deliveryEngine: AgentTriggerDeliveryEngine | undefined;+ let initializePromise: Promise<void> | undefined;+ let purgeRecoveryPromise: Promise<void> | undefined;+ let purgeRecoveryTimer: NodeJS.Timeout | undefined;+ let deliveryReady = false;+ let stopping = false;+ const isPrincipalActive = deps.isPrincipalActive;+ const host: AgentTriggerExecutionHost = createAgentTriggerExecutionHost({+ getBaseUrl: () => {+ const origin = process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin;+ if (origin == null) {+ throw new Error('Agent trigger service has not been initialized with a listener address');+ }+ return origin;+ },+ mintToken:+ deps.mintToken ??+ ((principal) => generateAgentTriggerToken(principal.userId, AGENT_TRIGGER_TOKEN_TTL)),+ ...(deps.fetch != null && { fetch: deps.fetch }),+ ...(deps.getTimezone != null && { getTimezone: deps.getTimezone }),+ ...(deps.timeoutMs != null && { timeoutMs: deps.timeoutMs }),+ });++ const requireMethods = (): AgentTriggerDeliveryPersistence => {+ if (deps.methods == null || !deliveryReady || isShutdownInProgress()) {+ throw new AgentTriggerServiceUnavailableError(+ 'Durable agent trigger delivery is not ready on this server',+ );+ }+ return deps.methods;+ };++ const requireCleanupMethods = (): AgentTriggerDeliveryPersistence => {+ if (deps.methods == null) {+ throw new AgentTriggerServiceUnavailableError(+ 'Durable agent trigger delivery is not configured on this server',+ );+ }+ return deps.methods;+ };++ const requireActivePrincipal = async (userId: string): Promise<void> => {+ if (isPrincipalActive != null && !(await runAsSystem(async () => isPrincipalActive(userId)))) {+ throw new AgentTriggerServiceUnavailableError(+ 'Agent trigger delivery principal is no longer active',+ );+ }+ };++ const dispatchForActivePrincipal = async (+ envelope: unknown,+ options?: { signal?: AbortSignal },+ ): Promise<AgentTriggerExecutionResult> => {+ const parsed = parseAgentTriggerEnvelope(envelope);+ await requireActivePrincipal(parsed.principal.userId);+ return host.dispatch(parsed, options);+ };++ const drainUser = async (userId: string): Promise<void> => {+ const methods = requireMethods();+ await deliveryEngine?.cancelUser(userId);++ try {+ const deadline = Date.now() + userDrainTimeoutMs;+ while (+ (await runAsSystem(async () =>+ methods.countActiveAgentTriggerDeliveriesByUser(userId, new Date()),+ )) > 0+ ) {+ if (Date.now() >= deadline) {+ throw new AgentTriggerServiceUnavailableError(+ `Timed out draining active agent trigger deliveries for user ${userId}`,+ );+ }+ await new Promise((resolve) => setTimeout(resolve, userDrainPollMs));+ }+ } finally {+ deliveryEngine?.releaseUserCancellation(userId);+ }+ };++ const recoverPurges = (): Promise<void> => {+ if (deps.methods == null || stopping) {+ return Promise.resolve();+ }+ if (purgeRecoveryPromise != null) {+ return purgeRecoveryPromise;+ }+ const methods = deps.methods;+ const current = runAsSystem(async () => {+ const [purgedUsers, publishedLanes, reclaimedLanes] = await Promise.all([+ methods.recoverAgentTriggerUserPurges(purgeRecoveryLimit),+ methods.recoverAgentTriggerLanePublications(purgeRecoveryLimit),+ methods.reclaimInactiveAgentTriggerLanes(purgeRecoveryLimit),+ ]);+ if (publishedLanes > 0) {+ deliveryEngine?.wake();+ }+ if (purgedUsers > 0 || publishedLanes > 0 || reclaimedLanes > 0) {+ logger.info('[agent-triggers] recovered durable delivery maintenance', {+ purgedUsers,+ publishedLanes,+ reclaimedLanes,+ });+ }+ })+ .catch((error) => {+ logger.error('[agent-triggers] durable delivery maintenance failed:', error);+ })+ .finally(() => {+ if (purgeRecoveryPromise === current) {+ purgeRecoveryPromise = undefined;+ }+ });+ purgeRecoveryPromise = current;+ return current;+ };++ const startPurgeRecovery = (): void => {+ if (purgeRecoveryTimer != null) {+ return;+ }+ void recoverPurges();+ purgeRecoveryTimer = setInterval(() => void recoverPurges(), purgeRecoveryIntervalMs);+ purgeRecoveryTimer.unref();+ };++ const stop = async (): Promise<void> => {+ stopping = true;+ deliveryReady = false;+ if (purgeRecoveryTimer != null) {+ clearInterval(purgeRecoveryTimer);+ purgeRecoveryTimer = undefined;+ }+ await initializePromise?.catch(() => undefined);+ await deliveryEngine?.stop();+ await purgeRecoveryPromise?.catch(() => undefined);+ };++ if (deps.methods != null) {+ registerShutdownTask('agent trigger delivery engine', stop, {+ phase: 'pre-drain',+ priority: 100,+ });+ }++ return {+ initialize: (options = {}) => {+ boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;+ if (deps.methods == null || deliveryReady) {+ return Promise.resolve();+ }+ if (initializePromise != null) {+ return initializePromise;+ }+ if (stopping || isShutdownInProgress()) {+ return Promise.reject(+ new AgentTriggerServiceUnavailableError(+ 'Durable agent trigger delivery cannot start during shutdown',+ ),+ );+ }+ const methods = deps.methods;+ initializePromise = runAsSystem(async () => {+ requireDeliveryOrigin(boundOrigin);+ await methods.ensureAgentTriggerDeliveryIndexes();+ if (stopping || isShutdownInProgress()) {+ throw new AgentTriggerServiceUnavailableError(+ 'Durable agent trigger delivery cannot start during shutdown',+ );+ }+ deliveryEngine = createAgentTriggerDeliveryEngine(+ {+ store: createDeliveryStore(methods),+ dispatch: dispatchForActivePrincipal,+ },+ deps.deliveryOptions,+ );+ deliveryReady = true;+ deliveryEngine.start();+ startPurgeRecovery();+ logger.info('[agent-triggers] durable delivery engine started');+ }).finally(() => {+ initializePromise = undefined;+ });+ return initializePromise;+ },+ stop,+ dispatch: dispatchForActivePrincipal,+ enqueue: async (envelope, options) => {+ const methods = requireMethods();+ const prepared = prepareAgentTriggerDelivery(envelope, options);+ await requireActivePrincipal(String(prepared.user));+ const queued = await runAsSystem(async () => methods.enqueueAgentTriggerDelivery(prepared));+ try {+ await requireActivePrincipal(String(prepared.user));+ } catch (error) {+ await drainUser(String(prepared.user));+ throw error;+ }+ deliveryEngine?.wake();+ return {+ id: queued.delivery.id,+ deliveryKey: queued.delivery.deliveryKey,+ status: queued.delivery.status,+ availableAt: queued.delivery.availableAt,+ replayed: queued.replayed,+ };+ },+ getDelivery: (deliveryKey) =>+ runAsSystem(async () => requireMethods().getAgentTriggerDelivery(deliveryKey)),+ getDeadLetters: (limit) =>+ runAsSystem(async () => requireMethods().getAgentTriggerDeadLetters(limit)),+ requeue: (id, availableAt = new Date()) =>+ runAsSystem(async () => requireMethods().requeueAgentTriggerDelivery(id, availableAt)),+ drainUser,+ prepareUserPurge: (userId, fenceStartedAt, tenantId) =>+ runAsSystem(async () =>+ requireCleanupMethods().prepareAgentTriggerUserPurge(userId, fenceStartedAt, tenantId),+ ),+ cancelUserPurge: (userId, fenceStartedAt) =>+ runAsSystem(async () =>+ requireCleanupMethods().cancelAgentTriggerUserPurge(userId, fenceStartedAt),+ ),+ // Account deletion may reach this post-commit cleanup after graceful+ // shutdown has begun. Persistence remains usable even though admissions+ // and the delivery engine are deliberately no longer ready.+ purgeUser: (userId) =>+ runAsSystem(async () => requireCleanupMethods().deleteAgentTriggerDeliveriesByUser(userId)),+ };+}diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.tsindex b5962a45f7..8dff0f8f9f 100644--- a/packages/api/src/app/index.ts+++ b/packages/api/src/app/index.ts@@ -7,5 +7,6 @@ export * from './checks';export * from './resolve';export * from './shutdown';export * from './server';+export * from './origin';export { resolveBuildInfo } from './build';export type { BuildInfo } from './build';diff --git a/packages/api/src/app/origin.ts b/packages/api/src/app/origin.tsnew file mode 100644index 0000000000..883111d38d--- /dev/null+++ b/packages/api/src/app/origin.ts@@ -0,0 +1,21 @@+/** A listener's `server.address()`. */+export interface BoundAddress {+ address: string;+ family?: string | number;+ port: number;+}++/** Resolve an HTTP origin that reaches the current process's bound listener. */+export function selfOriginFromAddress(+ address: BoundAddress | string | null | undefined,+): string | null {+ if (address == null || typeof address === 'string' || !address.port) {+ return null;+ }+ const bound = address.address;+ const isIPv6 = address.family === 'IPv6' || address.family === 6 || bound.includes(':');+ const loopback = isIPv6 ? '::1' : '127.0.0.1';+ const isWildcard = bound === '' || bound === '::' || bound === '0.0.0.0';+ const host = isWildcard ? loopback : bound;+ return `http://${isIPv6 ? `[${host}]` : host}:${address.port}`;+}diff --git a/packages/api/src/app/shutdown.spec.ts b/packages/api/src/app/shutdown.spec.tsindex 7733bb5fed..16f2de5c51 100644--- a/packages/api/src/app/shutdown.spec.ts+++ b/packages/api/src/app/shutdown.spec.ts@@ -2,6 +2,7 @@import http from 'http';import {setupGracefulShutdown,+ isShutdownInProgress,registerShutdownTask,__resetShutdownStateForTests,} from './shutdown';@@ -160,6 +161,26 @@ describe('setupGracefulShutdown', () => {expect(closeSpy).toHaveBeenCalledTimes(1);});+ it('exposes shutdown state before pre-drain work begins', async () => {+ let observedDuringPreDrain = false;+ jest.spyOn(server, 'close').mockImplementation(() => server);+ registerShutdownTask(+ 'observe-shutdown',+ () => {+ observedDuringPreDrain = isShutdownInProgress();+ },+ { phase: 'pre-drain' },+ );+ setupGracefulShutdown(server);++ expect(isShutdownInProgress()).toBe(false);+ triggerSignal('SIGTERM');+ await flush();++ expect(observedDuringPreDrain).toBe(true);+ expect(isShutdownInProgress()).toBe(true);+ });+it('force-exits with code 1 if shutdown exceeds the timeout', () => {jest.useFakeTimers();jest.spyOn(server, 'close').mockImplementation(() => server);diff --git a/packages/api/src/app/shutdown.ts b/packages/api/src/app/shutdown.tsindex dd131fb7d9..e8662296d5 100644--- a/packages/api/src/app/shutdown.ts+++ b/packages/api/src/app/shutdown.ts@@ -42,6 +42,11 @@ export function registerShutdownTask(tasks.push({ name, fn, phase, priority, registrationOrder: nextRegistrationOrder++ });}+/** Whether graceful shutdown has started, for admission paths that must fail closed. */+export function isShutdownInProgress(): boolean {+ return isShuttingDown;+}+/*** Wires SIGTERM, SIGINT, SIGQUIT, and SIGHUP to a graceful shutdown* sequence: initiate HTTP server close to stop accepting new connections,diff --git a/packages/api/src/crypto/jwt.spec.ts b/packages/api/src/crypto/jwt.spec.tsnew file mode 100644index 0000000000..ccf2d04ef8--- /dev/null+++ b/packages/api/src/crypto/jwt.spec.ts@@ -0,0 +1,52 @@+import {+ AGENT_TRIGGER_SCOPE,+ generateAgentTriggerToken,+ generateShortLivedToken,+ isAgentTriggerRequest,+} from './jwt';++function request(token: string, marker = true) {+ return {+ headers: {+ ...(marker && { 'x-lc-agent-trigger': '1' }),+ authorization: `Bearer ${token}`,+ },+ };+}++describe('agent trigger identity', () => {+ const original = process.env.JWT_SECRET;++ beforeAll(() => {+ process.env.JWT_SECRET = 'test-secret';+ });++ afterAll(() => {+ if (original == null) {+ delete process.env.JWT_SECRET;+ } else {+ process.env.JWT_SECRET = original;+ }+ });++ it('recognizes only a signed trigger scope with its transport marker', () => {+ const trigger = generateAgentTriggerToken('user-1');+ const ordinary = generateShortLivedToken('user-1');++ expect(isAgentTriggerRequest(request(trigger))).toBe(true);+ expect(isAgentTriggerRequest(request(trigger, false))).toBe(false);+ expect(isAgentTriggerRequest(request(ordinary))).toBe(false);+ expect(isAgentTriggerRequest(request('invalid'))).toBe(false);+ });++ it('uses the dedicated trigger scope without changing ordinary tokens', () => {+ const trigger = generateAgentTriggerToken('user-1');+ const payload = JSON.parse(Buffer.from(trigger.split('.')[1], 'base64url').toString()) as {+ id: string;+ scope: string;+ };++ expect(payload).toMatchObject({ id: 'user-1', scope: AGENT_TRIGGER_SCOPE });+ expect(generateShortLivedToken('user-1')).not.toBe(trigger);+ });+});diff --git a/packages/api/src/crypto/jwt.ts b/packages/api/src/crypto/jwt.tsindex 380fc613a1..db6d8140d3 100644--- a/packages/api/src/crypto/jwt.ts+++ b/packages/api/src/crypto/jwt.ts@@ -1,10 +1,17 @@import jwt from 'jsonwebtoken';+export const AGENT_TRIGGER_SCOPE = 'agent_trigger';++type AgentTriggerRequest = {+ headers?: Record<string, string | string[] | undefined>;+ _isAgentTrigger?: boolean;+};+/**- * Generate a short-lived JWT token- * @param {String} userId - The ID of the user- * @param {String} [expireIn='5m'] - The expiration time for the token (default is 5 minutes)- * @returns {String} - The generated JWT token+ * Generate a short-lived JWT token.+ * @param {String} userId - The ID of the user.+ * @param {String} [expireIn='5m'] - The expiration time for the token.+ * @returns {String} - The generated JWT token.*/export const generateShortLivedToken = (userId: string, expireIn: string = '5m'): string => {return jwt.sign({ id: userId }, process.env.JWT_SECRET!, {@@ -12,3 +19,33 @@ export const generateShortLivedToken = (userId: string, expireIn: string = '5m')algorithm: 'HS256',});};++/** Mint the server-only identity used by durable agent trigger admission. */+export const generateAgentTriggerToken = (userId: string, expireIn: string = '60s'): string => {+ return jwt.sign({ id: userId, scope: AGENT_TRIGGER_SCOPE }, process.env.JWT_SECRET!, {+ expiresIn: expireIn,+ algorithm: 'HS256',+ });+};++/** Verify the signed trigger scope together with its explicit transport marker. */+export const isAgentTriggerRequest = (req?: AgentTriggerRequest): boolean => {+ if (req?.headers?.['x-lc-agent-trigger'] !== '1') {+ return false;+ }+ const auth = req.headers.authorization;+ const token = typeof auth === 'string' && auth.startsWith('Bearer ') ? auth.slice(7) : undefined;+ if (token == null) {+ return false;+ }+ try {+ const payload = jwt.verify(token, process.env.JWT_SECRET!, { algorithms: ['HS256'] });+ return typeof payload === 'object' && payload.scope === AGENT_TRIGGER_SCOPE;+ } catch {+ return false;+ }+};++/** Skip the shared loopback IP bucket while retaining per-user and concurrency limits. */+export const exemptAgentTriggerFromIpLimiter = (req?: AgentTriggerRequest): boolean =>+ typeof req?._isAgentTrigger === 'boolean' ? req._isAgentTrigger : isAgentTriggerRequest(req);diff --git a/packages/api/src/crypto/limiters.spec.ts b/packages/api/src/crypto/limiters.spec.tsnew file mode 100644index 0000000000..2aa6f807f2--- /dev/null+++ b/packages/api/src/crypto/limiters.spec.ts@@ -0,0 +1,50 @@+import {+ exemptAgentTriggerFromIpLimiter,+ generateAgentTriggerToken,+ generateShortLivedToken,+ isAgentTriggerRequest,+} from './jwt';++describe('agent trigger IP limiter identity', () => {+ const original = process.env.JWT_SECRET;++ beforeAll(() => {+ process.env.JWT_SECRET = 'test-secret';+ });++ afterAll(() => {+ if (original == null) {+ delete process.env.JWT_SECRET;+ } else {+ process.env.JWT_SECRET = original;+ }+ });++ it('exempts only a signed trigger from the shared loopback IP bucket', () => {+ const token = generateAgentTriggerToken('user-1');+ const req = {+ headers: {+ 'x-lc-agent-trigger': '1',+ authorization: `Bearer ${token}`,+ },+ };++ expect(exemptAgentTriggerFromIpLimiter(req)).toBe(true);+ expect(exemptAgentTriggerFromIpLimiter({ headers: {} })).toBe(false);+ expect(+ exemptAgentTriggerFromIpLimiter({+ headers: {+ 'x-lc-agent-trigger': '1',+ authorization: `Bearer ${generateShortLivedToken('user-1')}`,+ },+ }),+ ).toBe(false);+ });++ it('retains the router-captured decision after the short-lived token expires', () => {+ const req = { headers: { authorization: 'Bearer expired' }, _isAgentTrigger: true };++ expect(isAgentTriggerRequest(req)).toBe(false);+ expect(exemptAgentTriggerFromIpLimiter(req)).toBe(true);+ });+});diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 9014d12602..5078051aa7 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -91,6 +91,8 @@ const REAPED_JOB_ERROR = 'Generation timed out';* digits; this trips only when Redis stalls, bounding buffered batches and* queued commands by pacing the producer instead of growing without limit. */const MAX_OUTSTANDING_COALESCED_RECEIPTS = 256;+const PROVIDER_DRAIN_TIMEOUT_MS = 30_000;+const PROVIDER_DRAIN_POLL_MS = 50;/** Bounded completed-request replay horizon. It exceeds the default 24-hour* approval window; if a custom/live job outlasts it, `resumeClaimedGeneration`@@ -1563,16 +1565,37 @@ class GenerationJobManagerClass {// resumed owner must acknowledge like any other live provider.const hadRunningProvider =receipt.status === 'running' && receipt.providerAbortReady !== false;- if (- await this.notifyReplacedGeneration(- streamId,- receipt.createdAt,- receipt.conversationId,- fallbackConversationId,- job.creationAttemptId,- hadRunningProvider,- )- ) {+ let delivered = await this.notifyReplacedGeneration(+ streamId,+ receipt.createdAt,+ receipt.conversationId,+ fallbackConversationId,+ job.creationAttemptId,+ hadRunningProvider,+ );+ if (delivered && receipt.providerDrained === false) {+ if (!receipt.providerExecutionId) {+ logger.error(+ `[GenerationJobManager] Replacement receipt lacks provider identity: ${streamId}/${receipt.createdAt}`,+ );+ delivered = false;+ } else {+ try {+ await this.waitForProviderExecutionDrain(+ streamId,+ receipt.createdAt,+ receipt.providerExecutionId,+ );+ } catch (error) {+ logger.error(+ `[GenerationJobManager] Replaced provider did not drain: ${streamId}/${receipt.createdAt}`,+ error,+ );+ delivered = false;+ }+ }+ }+ if (delivered) {acknowledged.push(receipt.createdAt);} else {allReceiptsDelivered = false;@@ -1844,61 +1867,76 @@ class GenerationJobManagerClass {* terminal hash that makes every duplicate retry wait until Redis TTL expiry. */private async terminalizeUnexposedGeneration(streamId: string,- job: Pick<SerializableJobData, 'createdAt' | 'conversationId'>,+ job: Pick<SerializableJobData, 'createdAt' | 'conversationId' | 'providerExecutionId'>,message: string,): Promise<boolean> {- const finalEvent: t.FinalEvent = {- final: true,- reconcile: true,- reconcileReason: 'terminal_payload_missing',- terminalStatus: 'error',- generationCreatedAt: job.createdAt,- conversation: { conversationId: job.conversationId ?? streamId },- };- let lastError: unknown;- for (let attempt = 0; attempt < 2; attempt++) {- try {- if (- await this.jobStore.transitionStatus(streamId, {- from: 'running',- to: 'error',- patch: {- completedAt: Date.now(),- error: message,- finalEvent: JSON.stringify(finalEvent),- },- expectCreatedAt: job.createdAt,- })- ) {- return true;+ try {+ const finalEvent: t.FinalEvent = {+ final: true,+ reconcile: true,+ reconcileReason: 'terminal_payload_missing',+ terminalStatus: 'error',+ generationCreatedAt: job.createdAt,+ conversation: { conversationId: job.conversationId ?? streamId },+ };+ let lastError: unknown;+ for (let attempt = 0; attempt < 2; attempt++) {+ try {+ if (+ await this.jobStore.transitionStatus(streamId, {+ from: 'running',+ to: 'error',+ patch: {+ completedAt: Date.now(),+ error: message,+ finalEvent: JSON.stringify(finalEvent),+ },+ expectCreatedAt: job.createdAt,+ })+ ) {+ return true;+ }+ } catch (error) {+ // As with create, the terminal CAS may have committed before the reply+ // was lost. Probe its exact epoch before retrying.+ lastError = error;}- } catch (error) {- // As with create, the terminal CAS may have committed before the reply- // was lost. Probe its exact epoch before retrying.- lastError = error;- }- try {- const current = await this.jobStore.getJob(streamId);- if (- current == null ||- current.createdAt !== job.createdAt ||- (current.status !== 'running' && current.status !== 'requires_action')- ) {- return true;+ try {+ const current = await this.jobStore.getJob(streamId);+ if (+ current == null ||+ current.createdAt !== job.createdAt ||+ (current.status !== 'running' && current.status !== 'requires_action')+ ) {+ return true;+ }+ } catch (error) {+ lastError = error;}- } catch (error) {- lastError = error;}- }- if (lastError != null) {- logger.error(- `[GenerationJobManager] Failed to terminalize unexposed generation ${streamId}:`,- lastError,- );+ if (lastError != null) {+ logger.error(+ `[GenerationJobManager] Failed to terminalize unexposed generation ${streamId}:`,+ lastError,+ );+ }+ return false;+ } finally {+ if (job.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.providerExecutionId,+ ).catch((error) => {+ logger.error(+ `[GenerationJobManager] Failed to mark unexposed provider drained ${streamId}:`,+ error,+ );+ });+ }}- return false;}async createJob(@@ -1938,7 +1976,12 @@ class GenerationJobManagerClass {const tenantId = getTenantId();const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;- const initialMetadata = sanitizeJobMetadata(options.initialMetadata ?? {});+ const creationAttemptId = randomUUID();+ const initialMetadata = {+ ...sanitizeJobMetadata(options.initialMetadata ?? {}),+ providerExecutionId: creationAttemptId,+ providerDrained: true,+ };if ((options.recoveredSteerId != null) !== (options.recoveredSteerPayload != null) ||(options.recoveredSteerPayload != null &&@@ -1946,7 +1989,6 @@ class GenerationJobManagerClass {) {throw new RecoveredSteerPayloadMismatchError();}- const creationAttemptId = randomUUID();// Capture the active epoch before the store atomically replaces it. A// subscriber attached to that predecessor filters events by generation,// so replacement must explicitly close/handoff that old attachment.@@ -1974,6 +2016,18 @@ class GenerationJobManagerClass {throw error;}if (error instanceof JobCreationSupersededError) {+ if (error.createdJob.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ error.createdJob.createdAt,+ error.createdJob.providerExecutionId,+ ).catch((drainError) => {+ logger.error(+ `[GenerationJobManager] Failed to mark superseded unexposed provider drained ${streamId}:`,+ drainError,+ );+ });+ }try {const current = (await this.jobStore.getJob(streamId)) as CreatedJobData | null;if (@@ -2092,6 +2146,13 @@ class GenerationJobManagerClass {currentRuntimeBeforeInstall.createdAt > jobData.createdAt) ||(ownedCreatedAtBeforeInstall != null && ownedCreatedAtBeforeInstall > jobData.createdAt)) {+ if (jobData.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ jobData.createdAt,+ jobData.providerExecutionId,+ );+ }throw new Error('Generation job was replaced during initialization');}if (this.shuttingDown) {@@ -2110,6 +2171,13 @@ class GenerationJobManagerClass {patch: { completedAt: Date.now(), error: SHUTDOWN_JOB_ERROR },expectCreatedAt: jobData.createdAt,});+ if (jobData.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ jobData.createdAt,+ jobData.providerExecutionId,+ );+ }const replacedLocal = this.runtimeState.get(streamId);if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {replacedLocal.abortController.abort();@@ -2186,6 +2254,13 @@ class GenerationJobManagerClass {runtimeImmediatelyBeforeInstall.createdAt > jobData.createdAt) ||(ownedImmediatelyBeforeInstall != null && ownedImmediatelyBeforeInstall > jobData.createdAt)) {+ if (jobData.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ jobData.createdAt,+ jobData.providerExecutionId,+ );+ }throw new Error('Generation job was replaced during initialization');}@@ -2301,6 +2376,18 @@ class GenerationJobManagerClass {finalizeError,);});+ if (jobData.providerExecutionId) {+ await this.markProviderExecutionDrained(+ streamId,+ jobData.createdAt,+ jobData.providerExecutionId,+ ).catch((drainError) => {+ logger.error(+ `[GenerationJobManager] Failed to mark partially initialized provider drained ${streamId}:`,+ drainError,+ );+ });+ }if (this.runtimeState.get(streamId) === runtime &&runtime.replacementTransportHold !== true@@ -2409,6 +2496,8 @@ class GenerationJobManagerClass {// Surface the owning replica's seal capability so the steer route can// honour it instead of probing its own (possibly older) SDK.preemptCapable: jobData.preemptCapable,+ providerExecutionId: jobData.providerExecutionId,+ providerDrained: jobData.providerDrained,steersClosed: jobData.steersClosed,idempotencyClientRequestId: jobData.idempotencyClientRequestId,terminalPersistencePending: jobData.terminalPersistencePending,@@ -3403,6 +3492,7 @@ class GenerationJobManagerClass {const terminalJob = await this.jobStore.getJob(streamId);if (!retainForTerminalReplay &&+ terminalJob?.providerDrained !== false &&(terminalJob?.createdAt !== createdAt || terminalJob.terminalPersistencePending !== true)) {// A same-stream replacement created after the claim makes this a safe@@ -3538,6 +3628,9 @@ class GenerationJobManagerClass {* same-stream replacement between authorization and the manager read* could be stopped by a stale request intended for its predecessor. */expectedCreatedAt?: number;+ /** Destructive user cleanup must wait until the provider owner has+ * completed every trailing persistence task, not merely received Stop. */+ awaitProviderDrain?: boolean;},): Promise<AbortResult> {const observedRuntime = this.runtimeState.get(streamId);@@ -3607,6 +3700,9 @@ class GenerationJobManagerClass {const abortableStatus = jobData.status;if (abortableStatus !== 'running' && abortableStatus !== 'requires_action') {+ if (options?.awaitProviderDrain) {+ await this.waitForProviderDrainIfRequired(streamId, jobData);+ }this.reconcileInactiveGeneration(streamId, jobData.createdAt, jobData, observedRuntime);logger.debug(`[GenerationJobManager] Cannot abort terminal job ${streamId}: ${jobData.status}`,@@ -3706,6 +3802,12 @@ class GenerationJobManagerClass {const jobStillActive =currentJob?.createdAt === jobData.createdAt &&(currentJob.status === 'running' || currentJob.status === 'requires_action');+ if (options?.awaitProviderDrain) {+ if (jobStillActive || currentJob?.createdAt !== jobData.createdAt) {+ throw new Error(`Failed to stop provider execution before user cleanup: ${streamId}`);+ }+ await this.waitForProviderDrainIfRequired(streamId, currentJob ?? jobData);+ }return {success: false,...(jobStillActive@@ -3864,7 +3966,13 @@ class GenerationJobManagerClass {...(publication.persistenceFailed && { persistenceFailed: true }),};} finally {- await this.finishTerminalJob(terminalClaim);+ try {+ if (options?.awaitProviderDrain) {+ await this.waitForProviderDrainIfRequired(streamId, jobData);+ }+ } finally {+ await this.finishTerminalJob(terminalClaim);+ }}}@@ -6212,6 +6320,107 @@ class GenerationJobManagerClass {await this.jobStore.updateJob(streamId, sanitizeJobMetadata(metadata), generationId);}+ /** Records that one exact provider segment has completed every trailing write.+ * The opaque segment id prevents a paused controller from acknowledging a+ * later HITL resume that reuses the same generation epoch. */+ async markProviderExecutionDrained(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ if (providerExecutionId.length === 0 || providerExecutionId.length > 128) {+ return false;+ }++ const [marked, recorded] = await Promise.all([+ this.jobStore.markProviderExecutionDrained?.(+ streamId,+ expectedCreatedAt,+ providerExecutionId,+ ) ?? Promise.resolve(false),+ this.eventTransport.recordProviderDrain?.(streamId, expectedCreatedAt, providerExecutionId) ??+ Promise.resolve(false),+ ]);+ const job = await this.jobStore.getJob(streamId);+ if (+ marked &&+ recorded &&+ this._cleanupOnComplete &&+ job?.createdAt === expectedCreatedAt &&+ job.providerExecutionId === providerExecutionId &&+ job.providerDrained === true &&+ job.terminalPersistencePending !== true &&+ (job.status === 'complete' || job.status === 'aborted')+ ) {+ await this.jobStore.deleteJob(streamId, expectedCreatedAt);+ }+ return marked;+ }++ /** Starts the initial provider owner only if account deletion, abort, or a+ * replacement has not already changed the exact running generation. */+ async beginProviderExecution(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ if (providerExecutionId.length === 0 || providerExecutionId.length > 128) {+ return false;+ }+ return (+ this.jobStore.beginProviderExecution?.(streamId, expectedCreatedAt, providerExecutionId) ??+ Promise.resolve(false)+ );+ }++ private async waitForProviderExecutionDrain(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<void> {+ const deadline = Date.now() + PROVIDER_DRAIN_TIMEOUT_MS;+ for (;;) {+ if (+ (await this.eventTransport+ .hasProviderDrain?.(streamId, expectedCreatedAt, providerExecutionId)+ .catch(() => false)) === true+ ) {+ return;+ }+ const job = await this.jobStore.getJob(streamId);+ if (+ job?.createdAt === expectedCreatedAt &&+ job.providerExecutionId === providerExecutionId &&+ job.providerDrained === true+ ) {+ return;+ }+ if (Date.now() >= deadline) {+ throw new Error(`Timed out waiting for provider execution to drain: ${streamId}`);+ }+ await new Promise<void>((resolve) => {+ setTimeout(resolve, PROVIDER_DRAIN_POLL_MS);+ });+ }+ }++ private async waitForProviderDrainIfRequired(+ streamId: string,+ job: SerializableJobData,+ ): Promise<void> {+ if (!job.providerExecutionId) {+ if (job.providerDrained !== false) {+ return;+ }+ throw new Error(`Cannot confirm provider execution drain for legacy job: ${streamId}`);+ }+ // Never trust the caller's pre-abort `providerDrained` snapshot here. The+ // provider can win its exact begin CAS after that read but before the+ // terminal CAS. Re-read the exact segment only after terminal ownership is+ // committed, when no not-yet-started provider can begin anymore.+ await this.waitForProviderExecutionDrain(streamId, job.createdAt, job.providerExecutionId);+ }+/*** Set reference to the graph's contentParts array.*/@@ -7122,6 +7331,12 @@ class GenerationJobManagerClass {return this.jobStore.getActiveJobIdsByUser(userId, tenantId);}+ /** Returns every generation whose provider can still mutate user-owned data,+ * including a terminal generation whose controller is finishing trailing writes. */+ async getCleanupBlockingJobIdsForUser(userId: string, tenantId?: string): Promise<string[]> {+ return this.jobStore.getCleanupBlockingJobIdsByUser(userId, tenantId);+ }+private async finalizeOwnedJobsForShutdown(): Promise<void> {const ownedJobs = [...this.ownedJobs];if (ownedJobs.length === 0) {diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.tsindex cd37e16f34..4e2e9e5c87 100644--- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts+++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts@@ -1147,6 +1147,28 @@ describe('RedisEventTransport', () => {}});+ it('persists provider drain proof under the exact generation and segment identity', async () => {+ const mockPublisher = createMockPublisher();+ const mockSubscriber = createMockSubscriber();+ const transport = new RedisEventTransport(+ mockPublisher as unknown as Redis,+ mockSubscriber as unknown as Redis,+ );+ const streamId = 'provider-drain-proof';++ await expect(transport.recordProviderDrain(streamId, 1234, 'segment-a')).resolves.toBe(true);+ expect(mockPublisher.set).toHaveBeenCalledWith(+ `stream:{${streamId}}:provider-drain:1234:segment-a`,+ '1',+ 'EX',+ 86400,+ );+ await expect(transport.hasProviderDrain(streamId, 1234, 'segment-a')).resolves.toBe(true);+ await expect(transport.hasProviderDrain(streamId, 1234, 'segment-b')).resolves.toBe(false);++ transport.destroy();+ });+it('waits for a delayed owner acknowledgement when cluster publish reports zero local receivers', async () => {const mockPublisher = createMockPublisher();const mockSubscriber = createMockSubscriber();diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.tsindex d23931e963..7197f88aba 100644--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts@@ -47,6 +47,54 @@ function jobHashFromCreationCall(call: unknown[]): Record<string, string> {}describe('RedisJobStore', () => {+ test('marks only the exact provider segment drained', async () => {+ const evalDrain = jest.fn().mockResolvedValue(1);+ const redis = {+ isCluster: true,+ eval: evalDrain,+ } as unknown as Cluster;+ const store = new RedisJobStore(redis);++ await expect(+ store.markProviderExecutionDrained('stream-provider-drain', 123456, 'segment-a'),+ ).resolves.toBe(true);++ const [script, keyCount, jobKey, createdAt, providerExecutionId] = evalDrain.mock.calls[0];+ expect(script).toContain('HGET", KEYS[1], "createdAt"');+ expect(script).toContain('HGET", KEYS[1], "providerExecutionId"');+ expect(script).toContain('HSET", KEYS[1], "providerDrained", "1"');+ expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([+ 1,+ 'stream:{stream-provider-drain}:job',+ '123456',+ 'segment-a',+ ]);+ });++ test('starts only the exact still-running initial provider segment', async () => {+ const evalBegin = jest.fn().mockResolvedValue(1);+ const redis = {+ isCluster: true,+ eval: evalBegin,+ } as unknown as Cluster;+ const store = new RedisJobStore(redis);++ await expect(+ store.beginProviderExecution('stream-provider-begin', 123456, 'segment-a'),+ ).resolves.toBe(true);++ const [script, keyCount, jobKey, createdAt, providerExecutionId] = evalBegin.mock.calls[0];+ expect(script).toContain('HGET", KEYS[1], "status") ~= "running"');+ expect(script).toContain('HGET", KEYS[1], "providerDrained") ~= "1"');+ expect(script).toContain('HSET", KEYS[1], "providerDrained", "0"');+ expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([+ 1,+ 'stream:{stream-provider-begin}:job',+ '123456',+ 'segment-a',+ ]);+ });+test('guards the atomic status transition with the expected creation epoch', async () => {const evalTransition = jest.fn().mockResolvedValue(0);const redis = {@@ -411,7 +459,18 @@ describe('RedisJobStore', () => {test('atomically resets predecessor state when creating a replacement', async () => {const evalJobCreation = jest.fn()- .mockImplementation((...args: unknown[]) => ['user-1', '', args[Number(args[1]) + 3]]);+ .mockImplementation((...args: unknown[]) => [+ 'user-1',+ '',+ args[Number(args[1]) + 3],+ '',+ '1',+ 'running',+ 'conversation-before-replacement',+ '1',+ 'provider-before-replacement',+ '0',+ ]);const redis = {isCluster: true,eval: evalJobCreation,@@ -423,7 +482,16 @@ describe('RedisJobStore', () => {const store = new RedisJobStore(redis);store.setCollectedUsage('stream-replacement', [{ input_tokens: 10 }]);- await store.createJob('stream-replacement', 'user-1');+ const replacement = await store.createJob('stream-replacement', 'user-1');++ expect(replacement.replacedJob).toMatchObject({+ createdAt: 1,+ status: 'running',+ conversationId: 'conversation-before-replacement',+ providerAbortReady: true,+ providerExecutionId: 'provider-before-replacement',+ providerDrained: false,+ });const [script, keyCount, ...args] = evalJobCreation.mock.calls[0];expect(script).toContain(diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.tsindex af05fea461..18777ca2e2 100644--- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts@@ -138,7 +138,13 @@ describe('RedisJobStore Integration Tests', () => {const store = new RedisJobStore(ioredisClient);await store.initialize();const streamId = `test-replaced-owner-${Date.now()}`;- const predecessor = await store.createJob(streamId, 'user-1', streamId);+ const providerExecutionId = 'provider-before-replacement';+ const predecessor = await store.createJob(streamId, 'user-1', streamId, undefined, {+ providerExecutionId,+ });+ await expect(+ store.beginProviderExecution(streamId, predecessor.createdAt, providerExecutionId),+ ).resolves.toBe(true);await store.transitionStatus(streamId, {from: 'running',to: 'requires_action',@@ -155,12 +161,20 @@ describe('RedisJobStore Integration Tests', () => {status: 'requires_action',conversationId: streamId,});+ expect(replacement.replacedJob).toMatchObject({+ providerExecutionId,+ providerDrained: false,+ });// The receipt is durably reconstructed for lost-reply recovery, but it// remains non-enumerable so ordinary job serializers cannot expose it.const durableReplacement = await store.getJob(streamId);expect((durableReplacement as typeof replacement).replacedJob).toEqual(replacement.replacedJob,);+ expect((durableReplacement as typeof replacement).replacedJob).toMatchObject({+ providerExecutionId,+ providerDrained: false,+ });expect(Object.getOwnPropertyDescriptor(durableReplacement!, 'replacedJob')).toMatchObject({enumerable: false,});diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.tsindex 1ebf2c7064..7c7b3da8d0 100644--- a/packages/api/src/stream/__tests__/startup.spec.ts+++ b/packages/api/src/stream/__tests__/startup.spec.ts@@ -137,6 +137,8 @@ describe('GenerationJobManager startup telemetry', () => {isTemporary: false,promptTokens: 0,discoveredTools: [],+ providerExecutionId: expect.any(String),+ providerDrained: true,});expect(job.metadata.tenantId).toBeUndefined();expect(job.metadata.pendingAction).toBeUndefined();@@ -150,6 +152,176 @@ describe('GenerationJobManager startup telemetry', () => {await manager.destroy();});+ it('retains a terminal generation until its exact provider segment has drained', async () => {+ const manager = createManager();+ const streamId = 'stream-provider-drain-retention';+ const job = await manager.createJob(streamId, 'user-1');+ const providerExecutionId = job.metadata.providerExecutionId!;+ await expect(+ manager.beginProviderExecution(streamId, job.createdAt, providerExecutionId),+ ).resolves.toBe(true);++ await expect(manager.abortJob(streamId)).resolves.toMatchObject({ success: true });+ await expect(manager.getActiveJobIdsForUser('user-1')).resolves.toEqual([]);+ await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([streamId]);+ await expect(manager.getJobStore().getJob(streamId)).resolves.toMatchObject({+ status: 'aborted',+ providerExecutionId,+ providerDrained: false,+ });++ await expect(+ manager.markProviderExecutionDrained(streamId, job.createdAt, 'different-segment'),+ ).resolves.toBe(false);+ await expect(manager.getJobStore().getJob(streamId)).resolves.toMatchObject({+ providerDrained: false,+ });++ await expect(+ manager.markProviderExecutionDrained(streamId, job.createdAt, providerExecutionId),+ ).resolves.toBe(true);+ await expect(manager.getJobStore().getJob(streamId)).resolves.toBeNull();+ await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);+ await manager.destroy();+ });++ it('keeps an undrained stale approval visible to destructive cleanup only', async () => {+ const manager = createManager();+ const streamId = 'stream-provider-drain-stale-approval';+ const job = await manager.createJob(streamId, 'user-1');+ await manager.beginProviderExecution(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );+ await manager.getJobStore().transitionStatus(streamId, {+ from: 'running',+ to: 'requires_action',+ expectCreatedAt: job.createdAt,+ });++ await expect(manager.getActiveJobIdsForUser('user-1')).resolves.toEqual([]);+ await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([streamId]);+ await manager.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );+ await manager.destroy();+ });++ it('waits for provider-owner drain proof before destructive abort returns', async () => {+ const manager = createManager();+ const streamId = 'stream-provider-drain-wait';+ const job = await manager.createJob(streamId, 'user-1');+ await expect(+ manager.beginProviderExecution(streamId, job.createdAt, job.metadata.providerExecutionId!),+ ).resolves.toBe(true);+ let abortSettled = false;+ const aborting = manager+ .abortJob(streamId, { awaitProviderDrain: true })+ .finally(() => (abortSettled = true));++ await new Promise<void>((resolve) => setImmediate(resolve));+ expect(job.abortController.signal.aborted).toBe(true);+ expect(abortSettled).toBe(false);++ await manager.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );+ await expect(aborting).resolves.toMatchObject({ success: true });+ expect(abortSettled).toBe(true);+ await manager.destroy();+ });++ it('finishes a destructive abort immediately when the provider never started', async () => {+ const manager = createManager();+ const streamId = 'stream-provider-never-started';+ await manager.createJob(streamId, 'user-1');++ await expect(manager.abortJob(streamId, { awaitProviderDrain: true })).resolves.toMatchObject({+ success: true,+ });+ await expect(manager.getJobStore().getJob(streamId)).resolves.toBeNull();+ await manager.destroy();+ });++ it('rechecks provider drain when provider startup races the destructive abort', async () => {+ const manager = createManager();+ const streamId = 'stream-provider-drain-start-race';+ const job = await manager.createJob(streamId, 'user-1');+ const jobStore = manager.getJobStore();+ const transition = jobStore.transitionStatusAndDrainSteers.bind(jobStore);+ jest+ .spyOn(jobStore, 'transitionStatusAndDrainSteers')+ .mockImplementationOnce(async (...args) => {+ await expect(+ manager.beginProviderExecution(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ ),+ ).resolves.toBe(true);+ return transition(...args);+ });++ let abortSettled = false;+ const aborting = manager+ .abortJob(streamId, { awaitProviderDrain: true })+ .finally(() => (abortSettled = true));++ await new Promise<void>((resolve) => setImmediate(resolve));+ expect(job.abortController.signal.aborted).toBe(true);+ expect(abortSettled).toBe(false);++ await manager.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );+ await expect(aborting).resolves.toMatchObject({ success: true });+ expect(abortSettled).toBe(true);+ await manager.destroy();+ });++ it('does not expose a replacement until its overwritten provider segment drains', async () => {+ const manager = createManager();+ const streamId = 'stream-provider-drain-replacement';+ const predecessor = await manager.createJob(streamId, 'user-1');+ await expect(+ manager.beginProviderExecution(+ streamId,+ predecessor.createdAt,+ predecessor.metadata.providerExecutionId!,+ ),+ ).resolves.toBe(true);+ let replacementSettled = false;+ const replacing = manager+ .createJob(streamId, 'user-1')+ .finally(() => (replacementSettled = true));++ await new Promise<void>((resolve) => setImmediate(resolve));+ expect(predecessor.abortController.signal.aborted).toBe(true);+ expect(replacementSettled).toBe(false);++ // The replacement already owns the job hash, so the predecessor can only+ // publish its exact cross-replica proof; it must not mark the new segment.+ await expect(+ manager.markProviderExecutionDrained(+ streamId,+ predecessor.createdAt,+ predecessor.metadata.providerExecutionId!,+ ),+ ).resolves.toBe(false);++ const replacement = await replacing;+ expect(replacement.createdAt).not.toBe(predecessor.createdAt);+ expect(replacementSettled).toBe(true);+ await manager.destroy();+ });+it('clears omitted per-turn identity when an in-memory job is replaced', async () => {const manager = createManager();await manager.createJob('stream-replaced-metadata', 'user-1', 'conversation-1', {@@ -703,6 +875,11 @@ describe('GenerationJobManager startup telemetry', () => {eventTransport.emitChunk(streamId, { data: { label: 'mismatched' } }, job.createdAt + 1);await expect(manager.abortJob(streamId)).resolves.toMatchObject({ success: true });+ await manager.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );await expect(jobStore.getJob(streamId)).resolves.toBeNull();expect(order).toEqual([]);@@ -2162,6 +2339,11 @@ describe('GenerationJobManager startup telemetry', () => {});expect(job.abortController.signal.aborted).toBe(true);expect(abortDisposer).toHaveBeenCalledTimes(1);+ await manager.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );await expect(jobStore.getJob(streamId)).resolves.toBeNull();await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([]);} finally {@@ -2890,6 +3072,11 @@ describe('GenerationJobManager startup telemetry', () => {expect.objectContaining({ aborted: true }),job.createdAt,);+ await manager.markProviderExecutionDrained(+ streamId,+ job.createdAt,+ job.metadata.providerExecutionId!,+ );await expect(jobStore.getJob(streamId)).resolves.toBeNull();expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);await manager.destroy();diff --git a/packages/api/src/stream/implementations/InMemoryEventTransport.ts b/packages/api/src/stream/implementations/InMemoryEventTransport.tsindex f9e953b9da..522e63fed4 100644--- a/packages/api/src/stream/implementations/InMemoryEventTransport.ts+++ b/packages/api/src/stream/implementations/InMemoryEventTransport.ts@@ -13,6 +13,18 @@ interface StreamState {*/export class InMemoryEventTransport implements IEventTransport {private streams = new Map<string, StreamState>();+ private providerDrainProofs = new Map<string, number>();+ private providerDrainWrites = 0;++ private static readonly PROVIDER_DRAIN_PROOF_TTL_MS = 5 * 60_000;++ private providerDrainKey(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): string {+ return `${streamId}:${generationId}:${providerExecutionId}`;+ }private getOrCreateStream(streamId: string): StreamState {let state = this.streams.get(streamId);@@ -110,6 +122,44 @@ export class InMemoryEventTransport implements IEventTransport {}}+ async recordProviderDrain(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ const now = Date.now();+ this.providerDrainWrites++;+ if (this.providerDrainWrites % 256 === 0) {+ for (const [key, expiresAt] of this.providerDrainProofs) {+ if (expiresAt <= now) {+ this.providerDrainProofs.delete(key);+ }+ }+ }+ this.providerDrainProofs.set(+ this.providerDrainKey(streamId, generationId, providerExecutionId),+ now + InMemoryEventTransport.PROVIDER_DRAIN_PROOF_TTL_MS,+ );+ return true;+ }++ async hasProviderDrain(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ const key = this.providerDrainKey(streamId, generationId, providerExecutionId);+ const expiresAt = this.providerDrainProofs.get(key);+ if (expiresAt == null) {+ return false;+ }+ if (expiresAt <= Date.now()) {+ this.providerDrainProofs.delete(key);+ return false;+ }+ return true;+ }+getSubscriberCount(streamId: string): number {const state = this.streams.get(streamId);return state?.emitter.listenerCount('chunk') ?? 0;@@ -186,6 +236,8 @@ export class InMemoryEventTransport implements IEventTransport {state.emitter.removeAllListeners();}this.streams.clear();+ this.providerDrainProofs.clear();+ this.providerDrainWrites = 0;logger.debug('[InMemoryEventTransport] Destroyed');}}diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.tsindex fc6bb4c8ed..a140491597 100644--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts@@ -285,6 +285,15 @@ export class InMemoryJobStore implements IJobStoreV2 {) {throw new Error('Invalid expected generation predecessor');}+ const providerExecutionId = initialMetadata.providerExecutionId;+ if (+ providerExecutionId != null &&+ (providerExecutionId.length === 0 || providerExecutionId.length > 128)+ ) {+ throw new Error('Invalid provider execution id');+ }+ const safeInitialMetadata = { ...initialMetadata };+ delete safeInitialMetadata.providerDrained;const assertOwnerCompatible = (): void => {const existingJob = this.jobs.get(streamId);if (@@ -467,11 +476,23 @@ export class InMemoryJobStore implements IJobStoreV2 {enumerable: false,});}+ if (previousJob.providerExecutionId != null) {+ Object.defineProperty(replaced, 'providerExecutionId', {+ value: previousJob.providerExecutionId,+ enumerable: false,+ });+ }+ if (previousJob.providerDrained != null) {+ Object.defineProperty(replaced, 'providerDrained', {+ value: previousJob.providerDrained,+ enumerable: false,+ });+ }addReplacedJob(replaced);}this.lastGenerationEpoch = createdAt;const job: CreatedJobData = {- ...initialMetadata,+ ...safeInitialMetadata,streamId,userId,...(tenantId && { tenantId }),@@ -485,6 +506,7 @@ export class InMemoryJobStore implements IJobStoreV2 {...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),...(recoveredSteerId !== undefined && { recoveredSteerId }),providerAbortReady: false,+ ...(providerExecutionId != null && { providerDrained: true }),syncSent: false,};if (creationAttemptId != null) {@@ -661,6 +683,37 @@ export class InMemoryJobStore implements IJobStoreV2 {Object.assign(job, updates);}+ async markProviderExecutionDrained(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ const job = this.jobs.get(streamId);+ if (job?.createdAt !== expectedCreatedAt || job.providerExecutionId !== providerExecutionId) {+ return false;+ }+ job.providerDrained = true;+ return true;+ }++ async beginProviderExecution(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ const job = this.jobs.get(streamId);+ if (+ job?.createdAt !== expectedCreatedAt ||+ job.providerExecutionId !== providerExecutionId ||+ job.status !== 'running' ||+ job.providerDrained !== true+ ) {+ return false;+ }+ job.providerDrained = false;+ return true;+ }+async finalizeTerminalPersistence(streamId: string,expectedCreatedAt: number,@@ -1174,6 +1227,18 @@ export class InMemoryJobStore implements IJobStoreV2 {* Also performs self-healing cleanup: removes stale entries for jobs that no longer exist.*/async getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {+ return this.getJobIdsByUser(userId, tenantId, false);+ }++ async getCleanupBlockingJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {+ return this.getJobIdsByUser(userId, tenantId, true);+ }++ private getJobIdsByUser(+ userId: string,+ tenantId: string | undefined,+ includeUndrained: boolean,+ ): string[] {const userKey = tenantId ? `${tenantId}:${userId}` : userId;const trackedIds = this.userJobMap.get(userKey);if (!trackedIds || trackedIds.size === 0) {@@ -1196,12 +1261,20 @@ export class InMemoryJobStore implements IJobStoreV2 {// A pending-approval job still occupies the user's conversation slot — but// only while its prompt is live: a past-`expiresAt` approval no longer// counts as active (cleanup/expiry will finalize it).- if (job.status === 'running' || job.status === 'requires_action') {- if (job.status === 'requires_action' && isPendingActionStale(job)) {+ if (+ job.status === 'running' ||+ job.status === 'requires_action' ||+ (includeUndrained && job.providerDrained === false)+ ) {+ if (+ job.status === 'requires_action' &&+ isPendingActionStale(job) &&+ !(includeUndrained && job.providerDrained === false)+ ) {continue;}activeIds.push(streamId);- } else {+ } else if (job.providerDrained !== false) {// Self-healing: job completed/deleted but mapping wasn't cleaned - fix it nowtrackedIds.delete(streamId);}diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.tsindex d172fb388c..c03d966eef 100644--- a/packages/api/src/stream/implementations/RedisEventTransport.ts+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts@@ -36,6 +36,9 @@ const KEYS = {/** Owner-issued proof that this exact generation processed an abort. */abortAck: (streamId: string, generationId: number) =>`stream:{${streamId}}:abort-ack:${generationId}`,+ /** Exact proof that a provider segment has fully unwound. */+ providerDrain: (streamId: string, generationId: number, providerExecutionId: string) =>+ `stream:{${streamId}}:provider-drain:${generationId}:${providerExecutionId}`,};/**@@ -206,6 +209,7 @@ const MAX_BUFFER_SIZE = 100;const GENERATION_EPOCH_GRACE_TTL_SECONDS = 300;/** Durable owner proof outlives receipt retries and process-local subscriptions. */const ABORT_ACK_TTL_SECONDS = 86400;+const PROVIDER_DRAIN_TTL_SECONDS = 86400;/*** Subscriber state for a stream@@ -1270,6 +1274,37 @@ export class RedisEventTransport implements IEventTransport {}}+ async recordProviderDrain(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ try {+ await this.publisher.set(+ KEYS.providerDrain(streamId, generationId, providerExecutionId),+ '1',+ 'EX',+ PROVIDER_DRAIN_TTL_SECONDS,+ );+ return true;+ } catch (error) {+ logger.error(`[RedisEventTransport] Failed to persist provider drain proof:`, error);+ return false;+ }+ }++ async hasProviderDrain(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ return (+ (await this.publisher.get(+ KEYS.providerDrain(streamId, generationId, providerExecutionId),+ )) === '1'+ );+ }+private async publishAbortAcknowledgement(streamId: string,generationId: number,diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.tsindex 2940a864c9..ec39812290 100644--- a/packages/api/src/stream/implementations/RedisJobStore.ts+++ b/packages/api/src/stream/implementations/RedisJobStore.ts@@ -329,7 +329,8 @@ const REPLACEMENT_RECEIPT_ACK_LUA =* ...hsetPairs]* Returns: [previousUserId | "", previousTenantId | "", createdAt, "",* replacedCreatedAt | "", replacedStatus | "", replacedConversationId | "",- * replacedProviderAbortReady | ""]+ * replacedProviderAbortReady | "", replacedProviderExecutionId | "",+ * replacedProviderDrained | ""]* Predecessor mismatch returns the latest retained epoch in the replaced* position plus an eighth active flag and ninth verified flag ("1" | "0").* Job-only metadata is empty when that epoch has outlived its hash. When all@@ -351,6 +352,8 @@ const JOB_CREATE_LUA ='local replacedStatus = redis.call("HGET", KEYS[1], "status") ' +'local replacedConversationId = redis.call("HGET", KEYS[1], "conversationId") ' +'local replacedProviderAbortReady = redis.call("HGET", KEYS[1], "providerAbortReady") ' ++ 'local replacedProviderExecutionId = redis.call("HGET", KEYS[1], "providerExecutionId") ' ++ 'local replacedProviderDrained = redis.call("HGET", KEYS[1], "providerDrained") ' +'local replacedProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") ' +'local MAX_SAFE_EPOCH = 9007199254740991 ' +'local function isSafeEpoch(value) return type(value) == "number" and value >= 0 ' +@@ -360,6 +363,12 @@ const JOB_CREATE_LUA ='local replacedEpoch = tonumber(replacedCreatedAt) local previousCreatedAt = replacedEpoch ' +'if previousJobExists == 1 and (not isSafeEpoch(replacedEpoch) or not isValidJobStatus(replacedStatus)) then ' +'return { "", "", "0", "replacement_receipt_corrupt" } end ' ++ 'if (replacedProviderExecutionId and not replacedProviderDrained) ' ++ 'or (replacedProviderDrained and not replacedProviderExecutionId) ' ++ 'or (replacedProviderExecutionId and (replacedProviderExecutionId == "" ' ++ 'or string.len(replacedProviderExecutionId) > 128)) ' ++ 'or (replacedProviderDrained and replacedProviderDrained ~= "0" and replacedProviderDrained ~= "1") then ' ++ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +'local retainedEpochRaw = redis.call("GET", KEYS[7]) local retainedEpoch = tonumber(retainedEpochRaw) ' +'if retainedEpochRaw and not isSafeEpoch(retainedEpoch) then ' +'return { "", "", "0", "generation_epoch_corrupt" } end ' +@@ -406,6 +415,10 @@ const JOB_CREATE_LUA ='or (previousJobExists == 1 and item.createdAt >= replacedEpoch) ' +'or (item.conversationId and type(item.conversationId) ~= "string") ' +'or (item.providerAbortReady ~= nil and type(item.providerAbortReady) ~= "boolean") ' ++ 'or (item.providerExecutionId ~= nil and (type(item.providerExecutionId) ~= "string" ' ++ 'or item.providerExecutionId == "" or string.len(item.providerExecutionId) > 128)) ' ++ 'or (item.providerDrained ~= nil and type(item.providerDrained) ~= "boolean") ' ++ 'or ((item.providerExecutionId ~= nil) ~= (item.providerDrained ~= nil)) ' +'or replacementSeen[tostring(item.createdAt)] then ' +'return { "", "", "0", "replacement_receipt_corrupt" } end ' +'lastReplacementEpoch = item.createdAt replacementSeen[tostring(item.createdAt)] = true ' +@@ -430,6 +443,8 @@ const JOB_CREATE_LUA ='local replaced = { createdAt = replacedEpoch, status = replacedStatus } ' +'if replacedConversationId then replaced.conversationId = replacedConversationId end ' +'if replacedProviderAbortReady then replaced.providerAbortReady = replacedProviderAbortReady == "1" end ' ++ 'if replacedProviderExecutionId then replaced.providerExecutionId = replacedProviderExecutionId end ' ++ 'if replacedProviderDrained then replaced.providerDrained = replacedProviderDrained == "1" end ' +'replacementChain[#replacementChain + 1] = replaced replacementSeen[tostring(replacedEpoch)] = true end ' +'local recoveredSteerId = ARGV[5] local expectedRecovery = nil ' +'if recoveredSteerId ~= "" and ARGV[10] ~= "2" then return { "", "", "0", "recovery_payload_mismatch" } end ' +@@ -533,7 +548,8 @@ const JOB_CREATE_LUA ='if claimTtl > 0 then redis.call("PEXPIRE", KEYS[10], claimTtl) end end ' +'return { previousUserId or "", previousTenantId or "", tostring(createdAt), "", ' +'replacedCreatedAt or "", replacedStatus or "", replacedConversationId or "", ' +- 'replacedProviderAbortReady or "" }';+ 'replacedProviderAbortReady or "", replacedProviderExecutionId or "", ' ++ 'replacedProviderDrained or "" }';/*** Epoch-guarded field update. Terminal writes reclaim same-slot content in the@@ -567,6 +583,23 @@ const JOB_UPDATE_LUA ='end ' +'return 1';+/** Exact provider-segment completion fence. A paused segment finishing after a+ * resume cannot mark the resumed provider drained because its opaque id differs. */+const PROVIDER_DRAIN_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'if redis.call("HGET", KEYS[1], "providerExecutionId") ~= ARGV[2] then return 0 end ' ++ 'redis.call("HSET", KEYS[1], "providerDrained", "1") return 1';++/** Exact initial provider-start fence. The controller rechecks account+ * deletion before this CAS; an abort/replacement that wins next prevents the+ * provider from starting after destructive cleanup has begun. */+const PROVIDER_BEGIN_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'if redis.call("HGET", KEYS[1], "providerExecutionId") ~= ARGV[2] then return 0 end ' ++ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return 0 end ' ++ 'if redis.call("HGET", KEYS[1], "providerDrained") ~= "1" then return 0 end ' ++ 'redis.call("HSET", KEYS[1], "providerDrained", "0") return 1';+/** Single-winner promotion from abort-persistence pending to a consumable* terminal payload. Owner success/failure and stale-owner recovery share this* CAS, so a timeout cannot overwrite a normal FINAL (or vice versa). */@@ -1757,6 +1790,15 @@ export class RedisJobStore implements IJobStoreV2 {) {throw new Error('Invalid expected generation predecessor');}+ const providerExecutionId = initialMetadata.providerExecutionId;+ if (+ providerExecutionId != null &&+ (providerExecutionId.length === 0 || providerExecutionId.length > 128)+ ) {+ throw new Error('Invalid provider execution id');+ }+ const safeInitialMetadata = { ...initialMetadata };+ delete safeInitialMetadata.providerDrained;let generationProtocolVersion: 1 | 2 = 1;if (initialMetadata.generationProtocolVersion === 1 ||@@ -1767,7 +1809,7 @@ export class RedisJobStore implements IJobStoreV2 {generationProtocolVersion = 2;}const job: CreatedJobData = {- ...initialMetadata,+ ...safeInitialMetadata,streamId,userId,...(tenantId && { tenantId }),@@ -1778,6 +1820,7 @@ export class RedisJobStore implements IJobStoreV2 {...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),...(recoveredSteerId !== undefined && { recoveredSteerId }),providerAbortReady: false,+ ...(providerExecutionId != null && { providerDrained: true }),syncSent: false,};if (creationAttemptId != null) {@@ -1909,6 +1952,18 @@ export class RedisJobStore implements IJobStoreV2 {previousOwner[7] !== ''? previousOwner[7] === '1': undefined;+ const replacedProviderExecutionId =+ Array.isArray(previousOwner) &&+ typeof previousOwner[8] === 'string' &&+ previousOwner[8] !== ''+ ? previousOwner[8]+ : undefined;+ const replacedProviderDrained =+ Array.isArray(previousOwner) &&+ typeof previousOwner[9] === 'string' &&+ previousOwner[9] !== ''+ ? previousOwner[9] === '1'+ : undefined;const replacedJob =replacedCreatedAt != null && Number.isFinite(replacedCreatedAt) && replacedStatus != null? {@@ -1925,6 +1980,18 @@ export class RedisJobStore implements IJobStoreV2 {enumerable: false,});}+ if (replacedJob != null && replacedProviderExecutionId != null) {+ Object.defineProperties(replacedJob, {+ providerExecutionId: {+ value: replacedProviderExecutionId,+ enumerable: false,+ },+ providerDrained: {+ value: replacedProviderDrained,+ enumerable: false,+ },+ });+ }const previousUserKeys =previousUserId !== ''? [KEYS.userJobs(previousUserId, previousTenantId || undefined)]@@ -2074,6 +2141,42 @@ export class RedisJobStore implements IJobStoreV2 {}}+ async markProviderExecutionDrained(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ return (+ Number(+ await this.redis.eval(+ PROVIDER_DRAIN_LUA,+ 1,+ KEYS.job(streamId),+ String(expectedCreatedAt),+ providerExecutionId,+ ),+ ) === 1+ );+ }++ async beginProviderExecution(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean> {+ return (+ Number(+ await this.redis.eval(+ PROVIDER_BEGIN_LUA,+ 1,+ KEYS.job(streamId),+ String(expectedCreatedAt),+ providerExecutionId,+ ),+ ) === 1+ );+ }+async finalizeTerminalPersistence(streamId: string,expectedCreatedAt: number,@@ -2103,7 +2206,8 @@ export class RedisJobStore implements IJobStoreV2 {left.createdAt === right.createdAt &&left.status === right.status &&left.userId === right.userId &&- left.tenantId === right.tenantId+ left.tenantId === right.tenantId &&+ left.providerDrained === right.providerDrained);}@@ -2119,7 +2223,10 @@ export class RedisJobStore implements IJobStoreV2 {observedUserKeys: Set<string>,): Promise<SerializableJobData | null> {const statusKey = job ? this.statusSetKey(job.status) : null;- const activeUserKey = job && statusKey != null ? KEYS.userJobs(job.userId, job.tenantId) : null;+ const activeUserKey =+ job && (statusKey != null || job.providerDrained === false)+ ? KEYS.userJobs(job.userId, job.tenantId)+ : null;if (this.isCluster) {const operations: Promise<unknown>[] = [@@ -2813,6 +2920,18 @@ export class RedisJobStore implements IJobStoreV2 {* @returns Array of conversation IDs with active jobs*/async getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {+ return this.getJobIdsByUser(userId, tenantId, false);+ }++ async getCleanupBlockingJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {+ return this.getJobIdsByUser(userId, tenantId, true);+ }++ private async getJobIdsByUser(+ userId: string,+ tenantId: string | undefined,+ includeUndrained: boolean,+ ): Promise<string[]> {const userJobsKey = KEYS.userJobs(userId, tenantId);const trackedIds = await this.redis.smembers(userJobsKey);@@ -2832,8 +2951,18 @@ export class RedisJobStore implements IJobStoreV2 {// polling and can complete.const belongsToUser =job?.userId === userId && (job.tenantId ?? undefined) === (tenantId ?? undefined);- if (belongsToUser && job && (job.status === 'running' || job.status === 'requires_action')) {- if (job.status === 'requires_action' && isPendingActionStale(job)) {+ if (+ belongsToUser &&+ job &&+ (job.status === 'running' ||+ job.status === 'requires_action' ||+ (includeUndrained && job.providerDrained === false))+ ) {+ if (+ job.status === 'requires_action' &&+ isPendingActionStale(job) &&+ !(includeUndrained && job.providerDrained === false)+ ) {continue;}activeIds.push(streamId);@@ -2851,8 +2980,14 @@ export class RedisJobStore implements IJobStoreV2 {if (currentBelongsToUser &¤tJob &&- (currentJob.status === 'running' || currentJob.status === 'requires_action') &&- !(currentJob.status === 'requires_action' && isPendingActionStale(currentJob))+ (currentJob.status === 'running' ||+ currentJob.status === 'requires_action' ||+ (includeUndrained && currentJob.providerDrained === false)) &&+ !(+ currentJob.status === 'requires_action' &&+ isPendingActionStale(currentJob) &&+ !(includeUndrained && currentJob.providerDrained === false)+ )) {activeIds.push(streamId);}@@ -4345,6 +4480,8 @@ export class RedisJobStore implements IJobStoreV2 {preemptCapable: data.preemptCapable != null ? data.preemptCapable === '1' : undefined,providerAbortReady:data.providerAbortReady != null ? data.providerAbortReady === '1' : undefined,+ providerExecutionId: data.providerExecutionId || undefined,+ providerDrained: data.providerDrained != null ? data.providerDrained === '1' : undefined,titleEvent: data.titleEvent || undefined,replayEvents: data.replayEvents || undefined,contextUsage: data.contextUsage || undefined,@@ -4407,6 +4544,12 @@ export class RedisJobStore implements IJobStoreV2 {(candidate.conversationId != null && typeof candidate.conversationId !== 'string') ||(candidate.providerAbortReady != null &&typeof candidate.providerAbortReady !== 'boolean') ||+ (candidate.providerExecutionId != null &&+ (typeof candidate.providerExecutionId !== 'string' ||+ candidate.providerExecutionId.length === 0 ||+ candidate.providerExecutionId.length > 128)) ||+ (candidate.providerDrained != null && typeof candidate.providerDrained !== 'boolean') ||+ (candidate.providerExecutionId != null) !== (candidate.providerDrained != null) ||seen.has(candidate.createdAt as number)) {throw new Error('Invalid generation replacement receipt');@@ -4426,6 +4569,16 @@ export class RedisJobStore implements IJobStoreV2 {enumerable: false,});}+ if (candidate.providerExecutionId != null) {+ Object.defineProperty(receipt, 'providerExecutionId', {+ value: candidate.providerExecutionId,+ enumerable: false,+ });+ Object.defineProperty(receipt, 'providerDrained', {+ value: candidate.providerDrained,+ enumerable: false,+ });+ }replacedJobs.push(receipt);}const scalarEpoch = scalarEpochRaw != null ? Number(scalarEpochRaw) : undefined;diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex ef49e2b472..1f1a5590f8 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -107,6 +107,13 @@ export interface SerializableJobData {* evidence and must be treated like true by replacement handoff. */providerAbortReady?: boolean;+ /** Opaque identity of the currently executing provider segment. A HITL resume+ * replaces it so an earlier paused segment cannot acknowledge the new run. */+ providerExecutionId?: string;+ /** False while the identified provider segment can still mutate user data;+ * true before provider startup and after the owner has fully unwound. */+ providerDrained?: boolean;+/** Whether the user-message created event has been emitted */createdEventEmitted?: boolean;@@ -201,7 +208,12 @@ export interface SerializableJobData {* reconstruct it without exposing it through normal job serialization. */export type ReplacedGeneration = Pick<SerializableJobData,- 'createdAt' | 'status' | 'conversationId' | 'providerAbortReady'+ | 'createdAt'+ | 'status'+ | 'conversationId'+ | 'providerAbortReady'+ | 'providerExecutionId'+ | 'providerDrained'>;/** Latest generation epoch checked by a conditional create. A retained epoch@@ -269,6 +281,8 @@ export type JobMetadataPatch = Partial<| 'discoveredTools'| 'activityPhaseSnapshot'| 'preemptCapable'+ | 'providerExecutionId'+ | 'providerDrained'| 'generationProtocolVersion'| 'resolvedAskUserQuestions'>@@ -651,7 +665,6 @@ export interface IJobStore {getJobCountByStatus(status: JobStatus): Promise<number>;destroy(): Promise<void>;getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]>;-setGraph(streamId: string, graph: StandardGraph, expectedCreatedAt?: number): void;setContentParts(streamId: string,@@ -750,6 +763,23 @@ export interface IJobStoreV2 extends IJobStore {expectedCreatedAt?: number,): Promise<void>;+ /** Atomically marks only the exact provider segment as fully unwound. */+ markProviderExecutionDrained(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean>;++ /** Activates the provider only while its exact generation is still running. */+ beginProviderExecution(+ streamId: string,+ expectedCreatedAt: number,+ providerExecutionId: string,+ ): Promise<boolean>;++ /** Includes terminal jobs whose provider still owns user-data writes. */+ getCleanupBlockingJobIdsByUser(userId: string, tenantId?: string): Promise<string[]>;+/** Atomically replaces an abort persistence-pending marker with the one* terminal payload that late subscribers may consume. Exactly one of the* owner, its failure path, or stale-owner recovery wins. */@@ -1277,6 +1307,18 @@ export interface IEventTransport {* A delayed replacement can use the proof after the owner's listeners retire. */recordAbortAcknowledgement?(streamId: string, generationId: number): Promise<boolean>;+ /** Persist/read exact proof that a provider segment can no longer mutate user data. */+ recordProviderDrain?(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): Promise<boolean>;+ hasProviderDrain?(+ streamId: string,+ generationId: number,+ providerExecutionId: string,+ ): Promise<boolean>;+/** Publish a predecessor DONE only while the current job's opaque creation* attempt still carries that predecessor in its durable receipt chain. */emitReplacedDoneConfirmed?(diff --git a/packages/api/src/stream/jobStoreCapabilities.ts b/packages/api/src/stream/jobStoreCapabilities.tsindex f3e1de0caf..2df4cb6bed 100644--- a/packages/api/src/stream/jobStoreCapabilities.ts+++ b/packages/api/src/stream/jobStoreCapabilities.ts@@ -10,6 +10,9 @@ import type { IJobStore, IJobStoreV2 } from './interfaces/IJobStore';*/export const JOB_STORE_V2_REQUIRED_METHODS = ['acknowledgeReplacedJobs',+ 'markProviderExecutionDrained',+ 'beginProviderExecution',+ 'getCleanupBlockingJobIdsByUser','finalizeTerminalPersistence','transitionStatusAndDrainSteers','takeoverIdempotencyKey',diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.tsindex 88ee943a8e..145df044b8 100644--- a/packages/api/src/types/stream.ts+++ b/packages/api/src/types/stream.ts@@ -41,6 +41,10 @@ export interface GenerationJobMetadata {activityPhaseSnapshot?: ActivityPhaseSnapshot;/** See `SerializableJobData.preemptCapable`. */preemptCapable?: boolean;+ /** Exact provider segment whose completion gates destructive user cleanup. */+ providerExecutionId?: string;+ /** False only while that exact provider segment can still mutate user data. */+ providerDrained?: boolean;/** Terminal close has atomically stopped new steer acceptance, even if the* final status CAS has not yet run. */steersClosed?: boolean;diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.tsindex fdb7fe28d9..9220c5f67a 100644--- a/packages/data-schemas/src/index.ts+++ b/packages/data-schemas/src/index.ts@@ -32,6 +32,7 @@ export {MAX_AUDIT_LOG_LIMIT,MAX_AUDIT_VERIFY_ROWS,MAX_TOOL_FAVORITES,+ AgentTriggerDeliveryConflictError,MCPAuthorityProofError,MAX_MCP_AUTHORITY_TARGETS,createMCPAuthorityBootRevision,diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.tsindex b5b3347bed..01468b7649 100644--- a/packages/data-schemas/src/methods/index.ts+++ b/packages/data-schemas/src/methods/index.ts@@ -93,6 +93,11 @@ import {type UpdateSkillResult,type ValidationIssue,} from './skill';+import {+ createAgentTriggerDeliveryMethods,+ AgentTriggerDeliveryConflictError,+ type AgentTriggerDeliveryMethods,+} from './triggerDelivery';import { createSkillSyncMethods, type SkillSyncMethods } from './skillSync';import type {SkillSyncStatusInput,@@ -147,6 +152,7 @@ export {};export { AUDIT_SCHEMA_VERSION, MAX_AUDIT_EXPORT_ROWS, MAX_AUDIT_LOG_LIMIT, MAX_AUDIT_VERIFY_ROWS };export { MAX_TOOL_FAVORITES };+export { AgentTriggerDeliveryConflictError };export type AllMethods = UserMethods &SessionMethods &@@ -182,6 +188,7 @@ export type AllMethods = UserMethods &PromptMethods &SkillMethods &SkillSyncMethods &+ AgentTriggerDeliveryMethods &AgentMethods &ConfigMethods &MCPAuthorityMethods;@@ -318,6 +325,7 @@ export function createMethods(...promptMethods,...skillMethods,...createSkillSyncMethods(mongoose),+ ...createAgentTriggerDeliveryMethods(mongoose),/* Tier 5 */...agentMethods,/* Config */@@ -375,6 +383,7 @@ export type {SkillSyncCredentialSummary,UpsertSkillSyncCredentialInput,SkillSyncMethods,+ AgentTriggerDeliveryMethods,AgentMethods,ConfigMethods,MCPAuthorityMethods,diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.tsnew file mode 100644index 0000000000..6e1e2dc292--- /dev/null+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts@@ -0,0 +1,956 @@+import mongoose from 'mongoose';+import { MongoMemoryServer } from 'mongodb-memory-server';+import type { Model } from 'mongoose';+import type {+ AgentTriggerDeliveryFailure,+ IAgentTriggerDeliveryDocument,+ IAgentTriggerLaneSequenceDocument,+ IAgentTriggerUserPurgeDocument,+} from '~/types/triggerDelivery';+import type { IUser } from '~/types/user';+import {+ AgentTriggerDeliveryConflictError,+ createAgentTriggerDeliveryMethods,+ type AgentTriggerDeliveryMethods,+} from './triggerDelivery';+import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';+import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';+import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';+import { createUserModel } from '../models/user';++jest.mock('~/config/winston', () => ({+ error: jest.fn(),+ warn: jest.fn(),+ info: jest.fn(),+ debug: jest.fn(),+}));++const DB_SETUP_TIMEOUT_MS = 60_000;+const START = new Date('2026-08-17T12:00:00.000Z');+let mongoServer: MongoMemoryServer;+let Delivery: Model<IAgentTriggerDeliveryDocument>;+let LaneSequence: Model<IAgentTriggerLaneSequenceDocument>;+let UserPurge: Model<IAgentTriggerUserPurgeDocument>;+let User: Model<IUser>;+let methods: AgentTriggerDeliveryMethods;+let counter = 0;++beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());+ Delivery = createAgentTriggerDeliveryModel(mongoose);+ LaneSequence = createAgentTriggerLaneSequenceModel(mongoose);+ UserPurge = createAgentTriggerUserPurgeModel(mongoose);+ User = createUserModel(mongoose);+ await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);+ methods = createAgentTriggerDeliveryMethods(mongoose);+}, DB_SETUP_TIMEOUT_MS);++afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+}, DB_SETUP_TIMEOUT_MS);++beforeEach(async () => {+ await Promise.all([+ Delivery.deleteMany({}),+ LaneSequence.deleteMany({}),+ UserPurge.deleteMany({}),+ User.deleteMany({}),+ ]);+});++function enqueueInput(+ overrides: Partial<Parameters<typeof methods.enqueueAgentTriggerDelivery>[0]> = {},+) {+ counter += 1;+ return {+ deliveryKey: `trigger_${counter}`,+ fingerprint: `fingerprint_${counter}`,+ orderingKey: 'ordering-1',+ envelope: { version: 1, delivery: counter },+ user: new mongoose.Types.ObjectId(),+ tenantId: 'tenant-1',+ availableAt: START,+ ...overrides,+ };+}++function transientFailure(overrides: Partial<AgentTriggerDeliveryFailure> = {}) {+ return {+ code: 'RATE_LIMITED',+ message: 'try later',+ certainty: 'definite' as const,+ retryable: true,+ attemptedAt: START,+ status: 429,+ ...overrides,+ };+}++describe('agent trigger delivery methods', () => {+ it('enqueues idempotently and rejects key reuse with different content', async () => {+ const input = enqueueInput();+ const first = await methods.enqueueAgentTriggerDelivery(input);+ const replay = await methods.enqueueAgentTriggerDelivery(input);++ expect(first.replayed).toBe(false);+ expect(first.delivery).toMatchObject({ status: 'pending', attempts: 0 });+ expect(replay).toMatchObject({ replayed: true, delivery: { id: first.delivery.id } });+ await expect(+ methods.enqueueAgentTriggerDelivery({ ...input, fingerprint: 'different' }),+ ).rejects.toBeInstanceOf(AgentTriggerDeliveryConflictError);+ expect(await Delivery.countDocuments()).toBe(1);+ });++ it('grants one atomic winner across concurrent claims', async () => {+ await methods.enqueueAgentTriggerDelivery(enqueueInput());+ const claims = await Promise.all(+ Array.from({ length: 8 }, (_, index) =>+ methods.claimNextAgentTriggerDelivery({+ workerId: `worker-${index}`,+ claimToken: `claim-${index}`,+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ }),+ ),+ );++ expect(claims.filter((claim) => claim != null)).toHaveLength(1);+ });++ it('allocates a replica-stable monotonic sequence within an ordering lane', async () => {+ const user = new mongoose.Types.ObjectId();+ const replicas = [+ createAgentTriggerDeliveryMethods(mongoose),+ createAgentTriggerDeliveryMethods(mongoose),+ ];+ const enqueued = await Promise.all(+ Array.from({ length: 12 }, (_, index) =>+ replicas[index % replicas.length].enqueueAgentTriggerDelivery(+ enqueueInput({ user, orderingKey: 'shared-lane' }),+ ),+ ),+ );++ expect(enqueued.map(({ delivery }) => delivery.laneSequence).sort((a, b) => a - b)).toEqual(+ Array.from({ length: 12 }, (_, index) => index + 1),+ );+ });++ it('publishes an abandoned lane owner before allocating the next sequence', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'publication-race';+ const firstInput = enqueueInput({ user, orderingKey, availableAt: START });+ const first = await Delivery.create({+ ...firstInput,+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });+ await LaneSequence.create({+ _id: orderingKey,+ value: 1,+ user,+ tailDeliveryId: first._id,+ publisherDeliveryId: first._id,+ publisherStartedAt: START,+ });++ const second = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({ user, orderingKey, availableAt: START }),+ );++ await expect(Delivery.findById(first._id).lean()).resolves.toMatchObject({+ status: 'pending',+ laneSequence: 1,+ });+ expect(second.delivery).toMatchObject({ status: 'pending', laneSequence: 2 });+ await expect(methods.findEarlierAgentTriggerDelivery(second.delivery)).resolves.toEqual({+ availableAt: START,+ });+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({ value: 2 });+ expect((await LaneSequence.findById(orderingKey).lean())?.publisherDeliveryId).toBeUndefined();+ });++ it('publishes an orphaned staging row before a later delivery in the lane', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'orphaned-staging';+ const firstInput = enqueueInput({ user, orderingKey, availableAt: START });+ const first = await Delivery.create({+ ...firstInput,+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });++ const second = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({ user, orderingKey, availableAt: START }),+ );++ await expect(Delivery.findById(first._id).lean()).resolves.toMatchObject({+ status: 'pending',+ laneSequence: 1,+ });+ expect(second.delivery).toMatchObject({ status: 'pending', laneSequence: 2 });+ await expect(methods.findEarlierAgentTriggerDelivery(second.delivery)).resolves.toEqual({+ availableAt: START,+ });+ });++ it('discovers an orphaned staging row during maintenance recovery', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'maintenance-staging';+ const staged = await Delivery.create({+ ...enqueueInput({ user, orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ stagingRecoveryAt: START,+ });++ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(1);+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({+ status: 'pending',+ laneSequence: 1,+ });+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({ value: 1 });+ });++ it('backfills and recovers a staging row created before the fairness cursor existed', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'legacy-maintenance-staging';+ const staged = await Delivery.create({+ ...enqueueInput({ user, orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });++ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(1);+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({+ status: 'pending',+ laneSequence: 1,+ });+ });++ it('leaves staging unpublished while its durable user purge marker exists', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'purge-fenced-staging';+ await UserPurge.create({ _id: user, fenceStartedAt: START, tenantId: 'tenant-1' });+ const staged = await Delivery.create({+ ...enqueueInput({ user, orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ stagingRecoveryAt: START,+ });++ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({+ status: 'staging',+ laneSequence: 0,+ });+ await expect(LaneSequence.findById(orderingKey)).resolves.toBeNull();+ });++ it('leaves an existing lane publisher fenced while its user purge marker exists', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'purge-fenced-publisher';+ await UserPurge.create({ _id: user, fenceStartedAt: START, tenantId: 'tenant-1' });+ const staged = await Delivery.create({+ ...enqueueInput({ user, orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });+ await LaneSequence.create({+ _id: orderingKey,+ value: 1,+ user,+ tailDeliveryId: staged._id,+ publisherDeliveryId: staged._id,+ publisherStartedAt: START,+ });++ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({+ status: 'staging',+ laneSequence: 0,+ });+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({+ publisherDeliveryId: staged._id,+ value: 1,+ });+ });++ it('rotates purge-fenced staging rows so later recovery work is not starved', async () => {+ const users = Array.from({ length: 3 }, () => new mongoose.Types.ObjectId());+ for (let index = 0; index < users.length; index += 1) {+ if (index < 2) {+ await UserPurge.create({ _id: users[index], fenceStartedAt: START });+ }+ await Delivery.create({+ ...enqueueInput({ user: users[index], orderingKey: `staging-rotation-${index}` }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ stagingRecoveryAt: new Date(START.getTime() + index),+ });+ }++ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(0);+ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(1);+ await expect(+ Delivery.findOne({ orderingKey: 'staging-rotation-2' }).lean(),+ ).resolves.toMatchObject({ status: 'pending', laneSequence: 1 });+ });++ it('rotates purge-fenced publishers so later recovery work is not starved', async () => {+ const users = Array.from({ length: 3 }, () => new mongoose.Types.ObjectId());+ for (let index = 0; index < users.length; index += 1) {+ if (index < 2) {+ await UserPurge.create({ _id: users[index], fenceStartedAt: START });+ }+ const orderingKey = `publisher-rotation-${index}`;+ const staged = await Delivery.create({+ ...enqueueInput({ user: users[index], orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ stagingRecoveryAt: START,+ });+ await LaneSequence.create({+ _id: orderingKey,+ value: 1,+ user: users[index],+ publisherDeliveryId: staged._id,+ publisherStartedAt: new Date(START.getTime() + index),+ });+ }++ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(0);+ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(1);+ await expect(+ Delivery.findOne({ orderingKey: 'publisher-rotation-2' }).lean(),+ ).resolves.toMatchObject({ status: 'pending', laneSequence: 1 });+ });++ it('abandons a lane acquired concurrently with a new purge marker', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'purge-race-staging';+ const staged = await Delivery.create({+ ...enqueueInput({ user, orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });+ const purgeExists = jest+ .spyOn(UserPurge, 'exists')+ .mockResolvedValueOnce(null)+ .mockResolvedValueOnce({ _id: user } as never);++ try {+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);+ } finally {+ purgeExists.mockRestore();+ }+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({+ status: 'staging',+ laneSequence: 0,+ });+ await expect(LaneSequence.findById(orderingKey)).resolves.toBeNull();+ });++ it('builds a sparse index for the periodic publisher recovery scan', async () => {+ const [laneIndexes, deliveryIndexes] = await Promise.all([+ LaneSequence.collection.indexes(),+ Delivery.collection.indexes(),+ ]);++ expect(laneIndexes).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ key: { publisherStartedAt: 1 }, sparse: true }),+ ]),+ );+ expect(deliveryIndexes).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ key: { stagingRecoveryAt: 1 }, sparse: true }),+ expect.objectContaining({ key: { laneCleanupPendingAt: 1 }, sparse: true }),+ ]),+ );+ });++ it('publishes an idempotent replay on its persisted ordering lane', async () => {+ const user = new mongoose.Types.ObjectId();+ const input = enqueueInput({ user, orderingKey: 'original-lane' });+ await Delivery.create({+ ...input,+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });++ const replay = await methods.enqueueAgentTriggerDelivery({+ ...input,+ orderingKey: 'changed-lane',+ });++ expect(replay).toMatchObject({+ replayed: true,+ delivery: { orderingKey: 'original-lane', laneSequence: 1, status: 'pending' },+ });+ await expect(LaneSequence.findById('original-lane').lean()).resolves.toMatchObject({+ value: 1,+ });+ await expect(LaneSequence.findById('changed-lane').lean()).resolves.toBeNull();+ });++ it('resumes its own abandoned publication without allocating a second sequence', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'publication-replay';+ const input = enqueueInput({ user, orderingKey, availableAt: START });+ const staged = await Delivery.create({+ ...input,+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });+ await LaneSequence.create({+ _id: orderingKey,+ value: 1,+ user,+ tailDeliveryId: staged._id,+ publisherDeliveryId: staged._id,+ publisherStartedAt: START,+ });++ const replay = await methods.enqueueAgentTriggerDelivery(input);++ expect(replay).toMatchObject({ replayed: true, delivery: { laneSequence: 1 } });+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({ value: 1 });+ });++ it('orders a lane by its atomic sequence instead of tied timestamps or process ObjectIds', async () => {+ const user = new mongoose.Types.ObjectId();+ const common = {+ orderingKey: 'tied-lane',+ envelope: { version: 1 },+ user,+ status: 'pending' as const,+ attempts: 0,+ availableAt: START,+ createdAt: START,+ updatedAt: START,+ };+ await Delivery.create([+ {+ ...common,+ _id: new mongoose.Types.ObjectId('ffffffffffffffffffffffff'),+ deliveryKey: 'first-by-sequence',+ fingerprint: 'first',+ laneSequence: 1,+ },+ {+ ...common,+ _id: new mongoose.Types.ObjectId('000000000000000000000001'),+ deliveryKey: 'second-by-sequence',+ fingerprint: 'second',+ laneSequence: 2,+ },+ ]);++ await expect(+ methods.findEarlierAgentTriggerDelivery({+ orderingKey: 'tied-lane',+ laneSequence: 2,+ }),+ ).resolves.toEqual({ availableAt: START });+ });++ it('honors the lease skew margin before takeover', async () => {+ await methods.enqueueAgentTriggerDelivery(enqueueInput());+ const first = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ expect(first).not.toBeNull();++ await expect(+ methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-2',+ claimToken: 'claim-2',+ now: new Date(START.getTime() + 70_000),+ leaseUntil: new Date(START.getTime() + 130_000),+ }),+ ).resolves.toBeNull();++ await expect(+ methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-2',+ claimToken: 'claim-2',+ now: new Date(START.getTime() + 91_000),+ leaseUntil: new Date(START.getTime() + 151_000),+ }),+ ).resolves.toMatchObject({ claimToken: 'claim-2', leaseBy: 'worker-2' });+ });++ it('fences every settlement against a stale claimant', async () => {+ await methods.enqueueAgentTriggerDelivery(enqueueInput());+ const first = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 10_000),+ });+ expect(first).not.toBeNull();+ const second = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-2',+ now: new Date(START.getTime() + 41_000),+ leaseUntil: new Date(START.getTime() + 101_000),+ });+ expect(second).not.toBeNull();++ await expect(+ methods.completeAgentTriggerDelivery({+ id: first!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: 1,+ result: { conversationId: 'stale' },+ settledAt: START,+ }),+ ).resolves.toBe(false);+ await expect(+ methods.beginAgentTriggerDeliveryAttempt({+ id: second!.id,+ workerId: 'worker-1',+ claimToken: 'claim-2',+ now: new Date(START.getTime() + 41_000),+ }),+ ).resolves.toBe(1);+ });++ it('blocks a due delivery behind an earlier pending item in its ordering lane', async () => {+ const first = await methods.enqueueAgentTriggerDelivery(+ enqueueInput({ availableAt: new Date(START.getTime() + 60_000) }),+ );+ await methods.enqueueAgentTriggerDelivery(enqueueInput());+ const claimed = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ expect(claimed?.id).not.toBe(first.delivery.id);++ await expect(methods.findEarlierAgentTriggerDelivery(claimed!)).resolves.toEqual({+ availableAt: new Date(START.getTime() + 60_000),+ });+ });++ it('records retry and success outcomes and expires only successful rows', async () => {+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput());+ const first = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: first!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ });+ await methods.retryAgentTriggerDelivery({+ id: first!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: attempt!,+ error: transientFailure(),+ availableAt: new Date(START.getTime() + 1_000),+ });++ const second = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-2',+ claimToken: 'claim-2',+ now: new Date(START.getTime() + 1_000),+ leaseUntil: new Date(START.getTime() + 61_000),+ });+ const secondAttempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: second!.id,+ workerId: 'worker-2',+ claimToken: 'claim-2',+ now: new Date(START.getTime() + 1_000),+ });+ await methods.completeAgentTriggerDelivery({+ id: second!.id,+ workerId: 'worker-2',+ claimToken: 'claim-2',+ attempt: secondAttempt!,+ result: { conversationId: 'conversation-1' },+ settledAt: new Date(START.getTime() + 2_000),+ });++ const stored = await methods.getAgentTriggerDelivery(queued.delivery.deliveryKey);+ expect(stored).toMatchObject({+ status: 'succeeded',+ attempts: 2,+ result: { conversationId: 'conversation-1' },+ history: [+ { attempt: 1, outcome: 'retry', workerId: 'worker-1' },+ { attempt: 2, outcome: 'succeeded', workerId: 'worker-2' },+ ],+ });+ expect(stored?.expiresAt?.getTime()).toBe(+ new Date(START.getTime() + 2_000).getTime() + 90 * 24 * 60 * 60_000,+ );+ });++ it('reclaims an inactive high-cardinality lane after its tail succeeds', async () => {+ const orderingKey = 'one-shot-resource-lane';+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ orderingKey }));+ const claimed = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: claimed!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ });++ await expect(+ methods.completeAgentTriggerDelivery({+ id: claimed!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: attempt!,+ result: { ok: true },+ settledAt: START,+ }),+ ).resolves.toBe(true);++ expect(await LaneSequence.countDocuments({ _id: orderingKey })).toBe(0);+ });++ it('rediscovers lane cleanup when success commits before its cleanup marker', async () => {+ const orderingKey = 'crashed-lane-cleanup';+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput({ orderingKey }));+ const claimed = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: claimed!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ });+ const laneUpdate = jest+ .spyOn(LaneSequence, 'updateOne')+ .mockRejectedValueOnce(new Error('down'));++ try {+ await expect(+ methods.completeAgentTriggerDelivery({+ id: claimed!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: attempt!,+ result: { ok: true },+ settledAt: START,+ }),+ ).resolves.toBe(true);+ } finally {+ laneUpdate.mockRestore();+ }+ await expect(Delivery.findById(queued.delivery.id).lean()).resolves.toMatchObject({+ status: 'succeeded',+ laneCleanupPendingAt: START,+ });++ await expect(methods.reclaimInactiveAgentTriggerLanes(1)).resolves.toBe(1);+ await expect(LaneSequence.findById(orderingKey)).resolves.toBeNull();+ expect(+ (await Delivery.findById(queued.delivery.id).lean())?.laneCleanupPendingAt,+ ).toBeUndefined();+ });++ it('restores the retry budget when dispatch is deferred before execution', async () => {+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput());+ const claim = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: claim!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ });++ await expect(+ methods.deferAgentTriggerDeliveryAttempt({+ id: claim!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: attempt!,+ availableAt: new Date(START.getTime() + 5_000),+ }),+ ).resolves.toBe(true);++ await expect(+ methods.getAgentTriggerDelivery(queued.delivery.deliveryKey),+ ).resolves.toMatchObject({+ status: 'pending',+ attempts: 0,+ availableAt: new Date(START.getTime() + 5_000),+ });+ });++ it('retains dead letters, truncates failure text, and supports explicit requeue', async () => {+ const user = new mongoose.Types.ObjectId();+ const orderingKey = 'dead-letter-requeue';+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user, orderingKey }));+ const claim = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({+ id: claim!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ });+ await methods.deadLetterAgentTriggerDelivery({+ id: claim!.id,+ workerId: 'worker-1',+ claimToken: 'claim-1',+ attempt: attempt!,+ error: transientFailure({+ code: 'X'.repeat(256),+ message: 'x'.repeat(3_000),+ retryable: false,+ }),+ settledAt: START,+ });++ const [dead] = await methods.getAgentTriggerDeadLetters();+ expect(dead).toMatchObject({ id: queued.delivery.id, status: 'dead' });+ expect(dead.lastError?.code).toHaveLength(128);+ expect(dead.lastError?.message).toHaveLength(2048);+ expect(dead.expiresAt).toBeUndefined();++ const later = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user, orderingKey }));+ const laterClaim = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-2',+ claimToken: 'claim-2',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ expect(laterClaim?.id).toBe(later.delivery.id);++ const stagedAhead = await Delivery.create({+ ...enqueueInput({ user, orderingKey }),+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ });+ await Delivery.collection.updateOne(+ { _id: stagedAhead._id },+ { $set: { updatedAt: new Date(0) } },+ );++ const requeued = await methods.requeueAgentTriggerDelivery(dead.id, START);+ expect(requeued).toMatchObject({+ status: 'pending',+ attempts: 0,+ requeueCount: 1,+ laneSequence: 4,+ });+ await expect(Delivery.findById(stagedAhead._id).lean()).resolves.toMatchObject({+ status: 'pending',+ laneSequence: 3,+ });+ expect(requeued?.lastError).toBeUndefined();+ await expect(methods.findEarlierAgentTriggerDelivery(requeued!)).resolves.toEqual({+ availableAt: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });+ expect(await methods.getAgentTriggerDeadLetters()).toEqual([]);+ await expect(methods.getAgentTriggerDeadLetters(Number.NaN)).rejects.toThrow(+ 'Agent trigger dead-letter limit must be a positive integer',+ );+ });++ it('counts only live leases while an account deletion drains', async () => {+ const user = new mongoose.Types.ObjectId();+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'claim-1',+ now: START,+ leaseUntil: new Date(START.getTime() + 60_000),+ });++ expect(await methods.countActiveAgentTriggerDeliveriesByUser(user, START)).toBe(1);+ expect(+ await methods.countActiveAgentTriggerDeliveriesByUser(+ user,+ new Date(START.getTime() + 60_001),+ ),+ ).toBe(0);+ });++ it('deletes all queued payloads for an erased user', async () => {+ const user = new mongoose.Types.ObjectId();+ const fenceStartedAt = new Date(START);+ await User.create({+ _id: user,+ email: '[email protected]',+ provider: 'local',+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ orderingKey: 'ordering-other' }));+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);+ await User.deleteOne({ _id: user });++ await methods.deleteAgentTriggerDeliveriesByUser(user);++ expect(await Delivery.countDocuments({ user })).toBe(0);+ expect(await Delivery.countDocuments()).toBe(1);+ expect(await LaneSequence.countDocuments({ user })).toBe(0);+ expect(await LaneSequence.countDocuments()).toBe(1);+ expect(await UserPurge.countDocuments({ _id: user })).toBe(0);+ });++ it('recovers an armed purge after the user deletion commits', async () => {+ const user = new mongoose.Types.ObjectId();+ const fenceStartedAt = new Date(START);+ await User.create({+ _id: user,+ email: '[email protected]',+ provider: 'local',+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt, 'tenant-1');++ await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(0);+ expect(await Delivery.countDocuments({ user })).toBe(1);+ expect(await UserPurge.countDocuments({ _id: user })).toBe(1);++ await User.deleteOne({ _id: user });+ await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(1);+ expect(await Delivery.countDocuments({ user })).toBe(0);+ expect(await LaneSequence.countDocuments({ user })).toBe(0);+ expect(await UserPurge.countDocuments({ _id: user })).toBe(0);+ });++ it('rotates active purge markers so a later committed purge is recovered', async () => {+ const users = Array.from({ length: 3 }, () => new mongoose.Types.ObjectId());+ for (const [index, user] of users.entries()) {+ await User.create({+ _id: user,+ email: `purge-rotation-${index}@example.com`,+ provider: 'local',+ agentTriggerDeletionStartedAt: START,+ });+ await methods.prepareAgentTriggerUserPurge(user, START);+ }+ await Delivery.create({+ ...enqueueInput({ user: users[2], orderingKey: 'committed-purge-lane' }),+ laneSequence: 1,+ status: 'pending',+ attempts: 0,+ requeueCount: 0,+ });+ await User.deleteOne({ _id: users[2] });++ await expect(methods.recoverAgentTriggerUserPurges(2)).resolves.toBe(0);+ await expect(methods.recoverAgentTriggerUserPurges(2)).resolves.toBe(1);+ expect(await UserPurge.countDocuments({ _id: users[2] })).toBe(0);+ expect(await Delivery.countDocuments({ user: users[2] })).toBe(0);+ expect(await UserPurge.countDocuments({ _id: { $in: users.slice(0, 2) } })).toBe(2);+ });++ it('disarms a stale purge marker without touching an active user delivery', async () => {+ const user = new mongoose.Types.ObjectId();+ const fenceStartedAt = new Date(START);+ await User.create({+ _id: user,+ email: '[email protected]',+ provider: 'local',+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);+ await User.updateOne({ _id: user }, { $unset: { agentTriggerDeletionStartedAt: 1 } });++ await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(0);+ expect(await Delivery.countDocuments({ user })).toBe(1);+ expect(await UserPurge.countDocuments({ _id: user })).toBe(0);+ });++ it('lets only the exact deletion owner cancel a prepared purge', async () => {+ const user = new mongoose.Types.ObjectId();+ const fenceStartedAt = new Date(START);+ await User.create({+ _id: user,+ email: '[email protected]',+ provider: 'local',+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);++ await expect(+ methods.cancelAgentTriggerUserPurge(user, new Date(START.getTime() + 1)),+ ).resolves.toBe(false);+ await expect(methods.cancelAgentTriggerUserPurge(user, fenceStartedAt)).resolves.toBe(true);+ });++ it('cannot disarm purge recovery after the user commit', async () => {+ const user = new mongoose.Types.ObjectId();+ const fenceStartedAt = new Date(START);+ await User.create({+ _id: user,+ email: '[email protected]',+ provider: 'local',+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);+ await User.deleteOne({ _id: user });++ await expect(methods.cancelAgentTriggerUserPurge(user, fenceStartedAt)).resolves.toBe(false);+ expect(await UserPurge.countDocuments({ _id: user })).toBe(1);+ });+});diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.tsnew file mode 100644index 0000000000..0e718caecc--- /dev/null+++ b/packages/data-schemas/src/methods/triggerDelivery.ts@@ -0,0 +1,995 @@+import type { Model, Types } from 'mongoose';+import type {+ AgentTriggerDeliveryClaim,+ AgentTriggerDeliveryFailure,+ AgentTriggerDeliveryRecord,+ AgentTriggerOrderingBlock,+ IAgentTriggerDelivery,+ IAgentTriggerDeliveryDocument,+ IAgentTriggerLaneSequence,+ IAgentTriggerLaneSequenceDocument,+ IAgentTriggerUserPurge,+ IAgentTriggerUserPurgeDocument,+} from '~/types/triggerDelivery';+import { createIndexesWithRetry } from '~/utils/retry';+import logger from '~/config/winston';++const DUPLICATE_KEY = 11000;+const LEASE_SKEW_MARGIN_MS = 30_000;+const SUCCESS_RETENTION_MS = 90 * 24 * 60 * 60_000;+const MAX_ERROR_CODE_LENGTH = 128;+const MAX_ERROR_MESSAGE_LENGTH = 2048;+const MAX_DEAD_LETTER_LIMIT = 200;+const DEFAULT_PURGE_RECOVERY_LIMIT = 25;+const MAX_PURGE_RECOVERY_LIMIT = 200;+const HISTORY_LIMIT = 64;++type DuplicateKeyError = { code?: number };++export class AgentTriggerDeliveryConflictError extends Error {+ constructor(deliveryKey: string) {+ super(`Agent trigger delivery key ${deliveryKey} was reused with different content`);+ this.name = 'AgentTriggerDeliveryConflictError';+ }+}++export interface EnqueueAgentTriggerDeliveryInput {+ deliveryKey: string;+ fingerprint: string;+ orderingKey: string;+ envelope: unknown;+ user: string | Types.ObjectId;+ tenantId?: string;+ availableAt: Date;+}++export interface AgentTriggerDeliveryFence {+ id: string;+ workerId: string;+ claimToken: string;+}++export interface AgentTriggerDeliveryMethods {+ ensureAgentTriggerDeliveryIndexes: () => Promise<void>;+ enqueueAgentTriggerDelivery: (+ input: EnqueueAgentTriggerDeliveryInput,+ ) => Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }>;+ claimNextAgentTriggerDelivery: (input: {+ workerId: string;+ claimToken: string;+ now: Date;+ leaseUntil: Date;+ }) => Promise<AgentTriggerDeliveryClaim | null>;+ findEarlierAgentTriggerDelivery: (+ delivery: Pick<AgentTriggerDeliveryRecord, 'orderingKey' | 'laneSequence'>,+ ) => Promise<AgentTriggerOrderingBlock | null>;+ releaseAgentTriggerDelivery: (+ input: AgentTriggerDeliveryFence & { availableAt: Date },+ ) => Promise<boolean>;+ beginAgentTriggerDeliveryAttempt: (+ input: AgentTriggerDeliveryFence & { now: Date },+ ) => Promise<number | null>;+ deferAgentTriggerDeliveryAttempt: (+ input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },+ ) => Promise<boolean>;+ completeAgentTriggerDelivery: (+ input: AgentTriggerDeliveryFence & {+ attempt: number;+ result: unknown;+ settledAt: Date;+ },+ ) => Promise<boolean>;+ retryAgentTriggerDelivery: (+ input: AgentTriggerDeliveryFence & {+ attempt: number;+ error: AgentTriggerDeliveryFailure;+ availableAt: Date;+ },+ ) => Promise<boolean>;+ deadLetterAgentTriggerDelivery: (+ input: AgentTriggerDeliveryFence & {+ attempt: number;+ error: AgentTriggerDeliveryFailure;+ settledAt: Date;+ },+ ) => Promise<boolean>;+ getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;+ getAgentTriggerDeadLetters: (limit?: number) => Promise<AgentTriggerDeliveryRecord[]>;+ requeueAgentTriggerDelivery: (+ id: string,+ availableAt: Date,+ ) => Promise<AgentTriggerDeliveryRecord | null>;+ countActiveAgentTriggerDeliveriesByUser: (+ user: string | Types.ObjectId,+ now: Date,+ ) => Promise<number>;+ recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;+ reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;+ prepareAgentTriggerUserPurge: (+ user: string | Types.ObjectId,+ fenceStartedAt: Date,+ tenantId?: string,+ ) => Promise<void>;+ cancelAgentTriggerUserPurge: (+ user: string | Types.ObjectId,+ fenceStartedAt: Date,+ ) => Promise<boolean>;+ recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;+ deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;+}++function normalizeFailure(error: AgentTriggerDeliveryFailure): AgentTriggerDeliveryFailure {+ return {+ ...error,+ code: error.code.slice(0, MAX_ERROR_CODE_LENGTH),+ message: error.message.slice(0, MAX_ERROR_MESSAGE_LENGTH),+ };+}++function toRecord(delivery: IAgentTriggerDelivery): AgentTriggerDeliveryRecord {+ if (delivery._id == null || delivery.createdAt == null) {+ throw new Error('Persisted agent trigger delivery is missing its identity or creation time');+ }+ return {+ id: String(delivery._id),+ deliveryKey: delivery.deliveryKey,+ fingerprint: delivery.fingerprint,+ orderingKey: delivery.orderingKey,+ laneSequence: delivery.laneSequence,+ envelope: delivery.envelope,+ user: delivery.user,+ status: delivery.status,+ attempts: delivery.attempts,+ availableAt: delivery.availableAt,+ createdAt: delivery.createdAt,+ ...(delivery.tenantId != null && { tenantId: delivery.tenantId }),+ ...(delivery.leaseBy != null && { leaseBy: delivery.leaseBy }),+ ...(delivery.leaseUntil != null && { leaseUntil: delivery.leaseUntil }),+ ...(delivery.claimToken != null && { claimToken: delivery.claimToken }),+ ...(delivery.lastError != null && { lastError: delivery.lastError }),+ ...(delivery.result !== undefined && { result: delivery.result }),+ ...(delivery.history != null && { history: delivery.history }),+ ...(delivery.settledAt != null && { settledAt: delivery.settledAt }),+ ...(delivery.expiresAt != null && { expiresAt: delivery.expiresAt }),+ ...(delivery.requeueCount != null && { requeueCount: delivery.requeueCount }),+ ...(delivery.updatedAt != null && { updatedAt: delivery.updatedAt }),+ };+}++function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliveryClaim | null {+ if (delivery == null) {+ return null;+ }+ const record = toRecord(delivery);+ if (+ record.status !== 'leased' ||+ record.claimToken == null ||+ record.leaseBy == null ||+ record.leaseUntil == null+ ) {+ throw new Error('Claimed agent trigger delivery is missing its lease fence');+ }+ return record as AgentTriggerDeliveryClaim;+}++export function createAgentTriggerDeliveryMethods(+ mongoose: typeof import('mongoose'),+): AgentTriggerDeliveryMethods {+ const Delivery = () =>+ mongoose.models.AgentTriggerDelivery as Model<IAgentTriggerDeliveryDocument>;+ const LaneSequence = () =>+ mongoose.models.AgentTriggerLaneSequence as Model<IAgentTriggerLaneSequenceDocument>;+ const UserPurge = () =>+ mongoose.models.AgentTriggerUserPurge as Model<IAgentTriggerUserPurgeDocument>;++ async function ensureAgentTriggerDeliveryIndexes(): Promise<void> {+ await Promise.all([+ createIndexesWithRetry(Delivery()),+ createIndexesWithRetry(LaneSequence()),+ createIndexesWithRetry(UserPurge()),+ ]);+ }++ async function abandonLanePublisher(lane: IAgentTriggerLaneSequence): Promise<boolean> {+ const publisherDeliveryId = lane.publisherDeliveryId;+ if (publisherDeliveryId == null) {+ return false;+ }+ if (!Number.isSafeInteger(lane.value) || lane.value <= 0) {+ throw new Error('Agent trigger lane publisher has an invalid sequence');+ }+ const publisherFence = {+ _id: lane._id,+ value: lane.value,+ publisherDeliveryId,+ };+ if (lane.value === 1) {+ const deleted = await LaneSequence().deleteOne(publisherFence);+ return deleted.deletedCount === 1;+ }+ const released = await LaneSequence().updateOne(publisherFence, {+ $inc: { value: -1 },+ $set: { cleanupRequestedAt: new Date() },+ $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },+ });+ return released.modifiedCount === 1;+ }++ async function recoverLanePublisher(lane: IAgentTriggerLaneSequence): Promise<boolean> {+ const publisherDeliveryId = lane.publisherDeliveryId;+ if (publisherDeliveryId == null) {+ return false;+ }+ if (!Number.isSafeInteger(lane.value) || lane.value <= 0) {+ throw new Error('Agent trigger lane publisher has an invalid sequence');+ }++ const published = await Delivery().updateOne(+ { _id: publisherDeliveryId, orderingKey: lane._id, status: 'staging' },+ {+ $set: { status: 'pending', laneSequence: lane.value },+ $unset: { stagingRecoveryAt: 1 },+ },+ );+ let publicationCommitted = published.modifiedCount === 1;+ if (published.modifiedCount === 0) {+ const current = await Delivery()+ .findById(publisherDeliveryId)+ .select('orderingKey laneSequence status')+ .lean<Pick<IAgentTriggerDelivery, 'orderingKey' | 'laneSequence' | 'status'>>();+ publicationCommitted =+ current != null &&+ current.orderingKey === lane._id &&+ current.status !== 'staging' &&+ current.laneSequence === lane.value;+ if (current?.orderingKey === lane._id && current.status === 'staging') {+ throw new Error('Failed to publish the reserved agent trigger delivery');+ }+ }++ const publisherFence = {+ _id: lane._id,+ value: lane.value,+ publisherDeliveryId,+ };+ if (publicationCommitted) {+ const released = await LaneSequence().updateOne(publisherFence, {+ $set: { tailDeliveryId: publisherDeliveryId },+ $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },+ });+ return published.modifiedCount === 1 || released.modifiedCount === 1;+ }++ // The selected row was published by a competing replica before this+ // reservation was acquired (or was removed by account cleanup). No later+ // reservation can exist while this fence is held, so the unused sequence+ // can be rolled back without creating a gap or regressing the lane tail.+ return abandonLanePublisher(lane);+ }++ async function publishStagedDelivery(+ delivery: IAgentTriggerDelivery,+ ): Promise<IAgentTriggerDelivery> {+ if (delivery._id == null) {+ throw new Error('Staged agent trigger delivery is missing its identity');+ }+ const deliveryId = delivery._id;+ const orderingKey = delivery.orderingKey;++ for (;;) {+ const current = await Delivery().findById(deliveryId).lean<IAgentTriggerDelivery>();+ if (current == null) {+ throw new Error('Staged agent trigger delivery disappeared before publication');+ }+ if (current.status !== 'staging') {+ return current;+ }+ if ((await UserPurge().exists({ _id: current.user })) != null) {+ return current;+ }++ const lane = await LaneSequence().findById(orderingKey).lean<IAgentTriggerLaneSequence>();+ if (lane?.publisherDeliveryId != null) {+ if ((await UserPurge().exists({ _id: lane.user })) != null) {+ return current;+ }+ await recoverLanePublisher(lane);+ continue;+ }++ // The durable staging row exists before sequence allocation. Publishing+ // the oldest visible row first lets any replica repair a writer that+ // stopped in that gap without allowing a later enqueue to overtake it.+ const next = await Delivery()+ .findOne({ orderingKey, status: 'staging' })+ // updatedAt is the current staging admission time: initial enqueue+ // sets it on insert, while explicit requeue refreshes it so an old+ // dead letter is admitted behind staging work that already exists.+ .sort({ updatedAt: 1, _id: 1 })+ .lean<IAgentTriggerDelivery>();+ if (next?._id == null) {+ continue;+ }++ let acquired: IAgentTriggerLaneSequence | null = null;+ try {+ acquired = await LaneSequence()+ .findOneAndUpdate(+ { _id: orderingKey, publisherDeliveryId: { $exists: false } },+ {+ $inc: { value: 1 },+ $set: {+ publisherDeliveryId: next._id,+ publisherStartedAt: new Date(),+ },+ $unset: { cleanupRequestedAt: 1 },+ $setOnInsert: {+ user: next.user,+ ...(next.tenantId != null && { tenantId: next.tenantId }),+ },+ },+ { new: true, upsert: true, setDefaultsOnInsert: true },+ )+ .lean<IAgentTriggerLaneSequence>();+ } catch (error) {+ if ((error as DuplicateKeyError)?.code !== DUPLICATE_KEY) {+ throw error;+ }+ }+ if (acquired == null) {+ continue;+ }++ // Account deletion creates this durable marker before it can delete+ // rows or lanes. Recheck after acquisition to close the cross-collection+ // race in which the marker appears between the first check and upsert.+ if ((await UserPurge().exists({ _id: next.user })) != null) {+ await abandonLanePublisher(acquired);+ return current;+ }++ await recoverLanePublisher(acquired);+ }+ }++ async function enqueueAgentTriggerDelivery(+ input: EnqueueAgentTriggerDeliveryInput,+ ): Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }> {+ let staged: IAgentTriggerDelivery;+ let replayed = false;+ try {+ const created = await Delivery().create({+ ...input,+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ stagingRecoveryAt: new Date(),+ });+ staged = created.toObject();+ } catch (error) {+ if ((error as DuplicateKeyError)?.code !== DUPLICATE_KEY) {+ throw error;+ }+ const existing = await Delivery()+ .findOne({ deliveryKey: input.deliveryKey })+ .lean<IAgentTriggerDelivery>();+ if (existing == null) {+ throw error;+ }+ if (existing.fingerprint !== input.fingerprint) {+ throw new AgentTriggerDeliveryConflictError(input.deliveryKey);+ }+ if (existing.status !== 'staging') {+ return { delivery: toRecord(existing), replayed: true };+ }+ staged = existing;+ replayed = true;+ }++ const published = await publishStagedDelivery(staged);+ return { delivery: toRecord(published), replayed };+ }++ /** Repairs abandoned reservations and staging rows left by crashed writers. */+ async function recoverAgentTriggerLanePublications(+ limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ ): Promise<number> {+ if (!Number.isSafeInteger(limit) || limit <= 0) {+ throw new TypeError('Agent trigger lane recovery limit must be a positive integer');+ }+ const boundedLimit = Math.min(limit, MAX_PURGE_RECOVERY_LIMIT);+ const lanes = await LaneSequence()+ .find({+ publisherDeliveryId: { $exists: true },+ publisherStartedAt: { $exists: true },+ })+ .sort({ publisherStartedAt: 1, _id: 1 })+ .limit(boundedLimit)+ .lean<IAgentTriggerLaneSequence[]>();+ let recovered = 0;+ const recoveryCursor = new Date();+ for (const lane of lanes) {+ if ((await UserPurge().exists({ _id: lane.user })) != null) {+ if (lane.publisherDeliveryId != null && lane.publisherStartedAt != null) {+ await LaneSequence().updateOne(+ {+ _id: lane._id,+ value: lane.value,+ publisherDeliveryId: lane.publisherDeliveryId,+ publisherStartedAt: lane.publisherStartedAt,+ },+ { $set: { publisherStartedAt: recoveryCursor } },+ { timestamps: false },+ );+ }+ continue;+ }+ if (await recoverLanePublisher(lane)) {+ recovered += 1;+ }+ }++ let remaining = boundedLimit - recovered;+ if (remaining <= 0) {+ return recovered;+ }+ /** Pre-cursor replicas can leave staging rows outside the sparse recovery+ * index. Backfill them in bounded batches before using the indexed scan. */+ const legacyStaged = await Delivery()+ .find({ status: 'staging', stagingRecoveryAt: { $exists: false } })+ .sort({ updatedAt: 1, _id: 1 })+ .limit(remaining)+ .lean<IAgentTriggerDelivery[]>();+ const legacyCursor = new Date();+ for (const delivery of legacyStaged) {+ if (delivery._id != null) {+ await Delivery().updateOne(+ { _id: delivery._id, status: 'staging', stagingRecoveryAt: { $exists: false } },+ { $set: { stagingRecoveryAt: legacyCursor } },+ { timestamps: false },+ );+ }+ }+ remaining -= legacyStaged.length;+ const legacyIds = legacyStaged.flatMap((delivery) =>+ delivery._id != null ? [delivery._id] : [],+ );+ const indexedStaged =+ remaining > 0+ ? await Delivery()+ .find({+ status: 'staging',+ stagingRecoveryAt: { $exists: true },+ ...(legacyIds.length > 0 && { _id: { $nin: legacyIds } }),+ })+ .sort({ stagingRecoveryAt: 1, _id: 1 })+ .limit(remaining)+ .lean<IAgentTriggerDelivery[]>()+ : [];+ const staged = [+ ...legacyStaged.map((delivery) => ({ ...delivery, stagingRecoveryAt: legacyCursor })),+ ...indexedStaged,+ ];+ const stagingRecoveryCursor = new Date();+ const rotateStagingRecovery = async (delivery: IAgentTriggerDelivery): Promise<void> => {+ if (delivery._id == null || delivery.stagingRecoveryAt == null) {+ return;+ }+ await Delivery().updateOne(+ {+ _id: delivery._id,+ status: 'staging',+ stagingRecoveryAt: delivery.stagingRecoveryAt,+ },+ { $set: { stagingRecoveryAt: stagingRecoveryCursor } },+ { timestamps: false },+ );+ };+ for (const delivery of staged) {+ try {+ const published = await publishStagedDelivery(delivery);+ if (published.status !== 'staging') {+ recovered += 1;+ } else {+ await rotateStagingRecovery(delivery);+ }+ } catch (error) {+ // Account deletion can remove a staged row while a replica is helping+ // it. Suppress only that confirmed race; operational failures remain+ // visible to the maintenance loop.+ if (delivery._id != null && (await Delivery().exists({ _id: delivery._id })) == null) {+ continue;+ }+ await rotateStagingRecovery(delivery);+ throw error;+ }+ }+ return recovered;+ }++ async function reclaimLaneIfInactive(orderingKey: string): Promise<boolean> {+ const lane = await LaneSequence().findById(orderingKey).lean<IAgentTriggerLaneSequence>();+ if (lane?.cleanupRequestedAt == null) {+ return false;+ }+ const stillRetained = await Delivery().exists({+ orderingKey,+ status: { $in: ['staging', 'pending', 'leased', 'dead'] },+ });+ if (stillRetained != null) {+ await LaneSequence().updateOne(+ { _id: orderingKey, cleanupRequestedAt: lane.cleanupRequestedAt },+ { $unset: { cleanupRequestedAt: 1 } },+ );+ return false;+ }+ const deleted = await LaneSequence().deleteOne({+ _id: orderingKey,+ ...(lane.tailDeliveryId == null+ ? { tailDeliveryId: { $exists: false } }+ : { tailDeliveryId: lane.tailDeliveryId }),+ cleanupRequestedAt: lane.cleanupRequestedAt,+ publisherDeliveryId: { $exists: false },+ });+ return deleted.deletedCount === 1;+ }++ async function fulfillLaneCleanupRequest(+ delivery: Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>,+ ): Promise<boolean> {+ if (delivery._id == null || delivery.laneCleanupPendingAt == null) {+ return false;+ }+ await LaneSequence().updateOne(+ { _id: delivery.orderingKey },+ { $set: { cleanupRequestedAt: delivery.laneCleanupPendingAt } },+ );+ await Delivery().updateOne(+ {+ _id: delivery._id,+ status: 'succeeded',+ laneCleanupPendingAt: delivery.laneCleanupPendingAt,+ },+ { $unset: { laneCleanupPendingAt: 1 } },+ { timestamps: false },+ );+ return reclaimLaneIfInactive(delivery.orderingKey);+ }++ /** Bounds high-cardinality ordering metadata after the final retained job settles. */+ async function reclaimInactiveAgentTriggerLanes(+ limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ ): Promise<number> {+ if (!Number.isSafeInteger(limit) || limit <= 0) {+ throw new TypeError('Agent trigger lane reclamation limit must be a positive integer');+ }+ const boundedLimit = Math.min(limit, MAX_PURGE_RECOVERY_LIMIT);+ const pendingCleanup = await Delivery()+ .find({ status: 'succeeded', laneCleanupPendingAt: { $exists: true } })+ .sort({ laneCleanupPendingAt: 1, _id: 1 })+ .limit(boundedLimit)+ .select('_id orderingKey laneCleanupPendingAt')+ .lean<Array<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>>();+ let reclaimed = 0;+ for (const delivery of pendingCleanup) {+ if (await fulfillLaneCleanupRequest(delivery)) {+ reclaimed += 1;+ }+ }++ const remaining = boundedLimit - pendingCleanup.length;+ if (remaining <= 0) {+ return reclaimed;+ }+ const lanes = await LaneSequence()+ .find({ cleanupRequestedAt: { $exists: true } })+ .sort({ cleanupRequestedAt: 1, _id: 1 })+ .limit(remaining)+ .select('_id')+ .lean<Array<Pick<IAgentTriggerLaneSequence, '_id'>>>();+ for (const lane of lanes) {+ if (await reclaimLaneIfInactive(lane._id)) {+ reclaimed += 1;+ }+ }+ return reclaimed;+ }++ async function claimNextAgentTriggerDelivery(input: {+ workerId: string;+ claimToken: string;+ now: Date;+ leaseUntil: Date;+ }): Promise<AgentTriggerDeliveryClaim | null> {+ const expiredBefore = new Date(input.now.getTime() - LEASE_SKEW_MARGIN_MS);+ const claimed = await Delivery()+ .findOneAndUpdate(+ {+ $or: [+ { status: 'pending', availableAt: { $lte: input.now } },+ { status: 'leased', leaseUntil: { $lte: expiredBefore } },+ ],+ },+ {+ $set: {+ status: 'leased',+ leaseBy: input.workerId,+ leaseUntil: input.leaseUntil,+ claimToken: input.claimToken,+ },+ $unset: { settledAt: 1, expiresAt: 1 },+ },+ { new: true, sort: { availableAt: 1, createdAt: 1, _id: 1 } },+ )+ .lean<IAgentTriggerDelivery>();+ return requireClaim(claimed);+ }++ async function findEarlierAgentTriggerDelivery(+ delivery: Pick<AgentTriggerDeliveryRecord, 'orderingKey' | 'laneSequence'>,+ ): Promise<AgentTriggerOrderingBlock | null> {+ const earlier = await Delivery()+ .findOne({+ orderingKey: delivery.orderingKey,+ status: { $in: ['pending', 'leased'] },+ laneSequence: { $lt: delivery.laneSequence },+ })+ .sort({ laneSequence: 1 })+ .select('availableAt leaseUntil')+ .lean<Pick<IAgentTriggerDelivery, 'availableAt' | 'leaseUntil'>>();+ if (earlier == null) {+ return null;+ }+ return {+ availableAt: earlier.availableAt,+ ...(earlier.leaseUntil != null && { leaseUntil: earlier.leaseUntil }),+ };+ }++ const fence = (input: AgentTriggerDeliveryFence) => ({+ _id: input.id,+ status: 'leased',+ leaseBy: input.workerId,+ claimToken: input.claimToken,+ });++ async function releaseAgentTriggerDelivery(+ input: AgentTriggerDeliveryFence & { availableAt: Date },+ ): Promise<boolean> {+ const result = await Delivery().updateOne(fence(input), {+ $set: { status: 'pending', availableAt: input.availableAt },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },+ });+ return result.modifiedCount === 1;+ }++ async function beginAgentTriggerDeliveryAttempt(+ input: AgentTriggerDeliveryFence & { now: Date },+ ): Promise<number | null> {+ const updated = await Delivery()+ .findOneAndUpdate(+ { ...fence(input), leaseUntil: { $gt: input.now } },+ { $inc: { attempts: 1 } },+ { new: true },+ )+ .select('attempts')+ .lean<Pick<IAgentTriggerDelivery, 'attempts'>>();+ return updated?.attempts ?? null;+ }++ /** Releases a pre-dispatch deferral and restores the attempt consumed by beginAttempt. */+ async function deferAgentTriggerDeliveryAttempt(+ input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },+ ): Promise<boolean> {+ if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) {+ throw new TypeError('attempt must be a positive integer');+ }+ const result = await Delivery().updateOne(+ { ...fence(input), attempts: input.attempt },+ {+ $inc: { attempts: -1 },+ $set: { status: 'pending', availableAt: input.availableAt },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },+ },+ );+ return result.modifiedCount === 1;+ }++ async function completeAgentTriggerDelivery(+ input: AgentTriggerDeliveryFence & {+ attempt: number;+ result: unknown;+ settledAt: Date;+ },+ ): Promise<boolean> {+ const completed = await Delivery()+ .findOneAndUpdate(+ fence(input),+ {+ $set: {+ status: 'succeeded',+ result: input.result,+ settledAt: input.settledAt,+ expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),+ laneCleanupPendingAt: input.settledAt,+ },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, lastError: 1 },+ $push: {+ history: {+ $each: [+ {+ attempt: input.attempt,+ outcome: 'succeeded',+ at: input.settledAt,+ workerId: input.workerId,+ },+ ],+ $slice: -HISTORY_LIMIT,+ },+ },+ },+ { new: true },+ )+ .select('_id orderingKey laneCleanupPendingAt')+ .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();+ if (completed?._id == null) {+ return false;+ }++ try {+ await fulfillLaneCleanupRequest(completed);+ } catch (error) {+ // Delivery success is authoritative; counter reclamation is safe to retry later.+ logger.warn('[agent-triggers] failed to reclaim an inactive ordering lane', {+ orderingKey: completed.orderingKey,+ error: error instanceof Error ? error.message : String(error),+ });+ }+ return true;+ }++ async function retryAgentTriggerDelivery(+ input: AgentTriggerDeliveryFence & {+ attempt: number;+ error: AgentTriggerDeliveryFailure;+ availableAt: Date;+ },+ ): Promise<boolean> {+ const error = normalizeFailure(input.error);+ const result = await Delivery().updateOne(fence(input), {+ $set: { status: 'pending', availableAt: input.availableAt, lastError: error },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, settledAt: 1, expiresAt: 1 },+ $push: {+ history: {+ $each: [+ {+ attempt: input.attempt,+ outcome: 'retry',+ at: error.attemptedAt,+ workerId: input.workerId,+ error,+ },+ ],+ $slice: -HISTORY_LIMIT,+ },+ },+ });+ return result.modifiedCount === 1;+ }++ async function deadLetterAgentTriggerDelivery(+ input: AgentTriggerDeliveryFence & {+ attempt: number;+ error: AgentTriggerDeliveryFailure;+ settledAt: Date;+ },+ ): Promise<boolean> {+ const error = normalizeFailure(input.error);+ const result = await Delivery().updateOne(fence(input), {+ $set: { status: 'dead', settledAt: input.settledAt, lastError: error },+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },+ $push: {+ history: {+ $each: [+ {+ attempt: input.attempt,+ outcome: 'dead',+ at: input.settledAt,+ workerId: input.workerId,+ error,+ },+ ],+ $slice: -HISTORY_LIMIT,+ },+ },+ });+ return result.modifiedCount === 1;+ }++ async function getAgentTriggerDelivery(+ deliveryKey: string,+ ): Promise<AgentTriggerDeliveryRecord | null> {+ const delivery = await Delivery().findOne({ deliveryKey }).lean<IAgentTriggerDelivery>();+ return delivery == null ? null : toRecord(delivery);+ }++ async function getAgentTriggerDeadLetters(limit = 50): Promise<AgentTriggerDeliveryRecord[]> {+ if (!Number.isSafeInteger(limit) || limit <= 0) {+ throw new TypeError('Agent trigger dead-letter limit must be a positive integer');+ }+ const boundedLimit = Math.min(limit, MAX_DEAD_LETTER_LIMIT);+ const deliveries = await Delivery()+ .find({ status: 'dead' })+ .sort({ updatedAt: -1, _id: -1 })+ .limit(boundedLimit)+ .lean<IAgentTriggerDelivery[]>();+ return deliveries.map(toRecord);+ }++ async function requeueAgentTriggerDelivery(+ id: string,+ availableAt: Date,+ ): Promise<AgentTriggerDeliveryRecord | null> {+ const staged = await Delivery()+ .findOneAndUpdate(+ { _id: id, status: 'dead' },+ {+ $set: {+ status: 'staging',+ laneSequence: 0,+ attempts: 0,+ availableAt,+ stagingRecoveryAt: new Date(),+ },+ $unset: {+ leaseBy: 1,+ leaseUntil: 1,+ claimToken: 1,+ lastError: 1,+ result: 1,+ settledAt: 1,+ expiresAt: 1,+ laneCleanupPendingAt: 1,+ },+ $inc: { requeueCount: 1 },+ },+ { new: true },+ )+ .lean<IAgentTriggerDelivery>();+ if (staged == null) {+ return null;+ }++ // Requeue is a new lane admission. Retaining the dead letter's original+ // sequence could let it overlap a newer delivery that is already running.+ return toRecord(await publishStagedDelivery(staged));+ }++ async function countActiveAgentTriggerDeliveriesByUser(+ user: string | Types.ObjectId,+ now: Date,+ ): Promise<number> {+ return Delivery().countDocuments({ user, status: 'leased', leaseUntil: { $gt: now } });+ }++ function requireValidFence(fenceStartedAt: Date): void {+ if (!(fenceStartedAt instanceof Date) || !Number.isFinite(fenceStartedAt.getTime())) {+ throw new TypeError('fenceStartedAt must be a valid Date');+ }+ }++ /** Arms independently retryable cleanup before the user deletion can commit. */+ async function prepareAgentTriggerUserPurge(+ user: string | Types.ObjectId,+ fenceStartedAt: Date,+ tenantId?: string,+ ): Promise<void> {+ requireValidFence(fenceStartedAt);+ const ownsFence = await mongoose.models.User.exists({+ _id: user,+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ if (ownsFence == null) {+ throw new Error('Cannot prepare trigger purge without owning the user deletion fence');+ }+ await UserPurge().updateOne(+ { _id: user },+ {+ $set: {+ fenceStartedAt,+ ...(tenantId != null && { tenantId }),+ },+ },+ { upsert: true },+ );+ }++ /** Disarms only the pre-commit deletion attempt that owns this marker. */+ async function cancelAgentTriggerUserPurge(+ user: string | Types.ObjectId,+ fenceStartedAt: Date,+ ): Promise<boolean> {+ requireValidFence(fenceStartedAt);+ const ownsLiveFence = await mongoose.models.User.exists({+ _id: user,+ agentTriggerDeletionStartedAt: fenceStartedAt,+ });+ if (ownsLiveFence == null) {+ return false;+ }+ const result = await UserPurge().deleteOne({ _id: user, fenceStartedAt });+ return result.deletedCount === 1;+ }++ /** Recovers cleanup markers whose users are gone; active-user markers are never destructive. */+ async function recoverAgentTriggerUserPurges(+ limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ ): Promise<number> {+ if (!Number.isSafeInteger(limit) || limit <= 0) {+ throw new TypeError('Agent trigger purge recovery limit must be a positive integer');+ }+ const markers = await UserPurge()+ .find({})+ .sort({ updatedAt: 1, _id: 1 })+ .limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))+ .lean<IAgentTriggerUserPurge[]>();+ let recovered = 0;+ for (const marker of markers) {+ const user = await mongoose.models.User.findById(marker._id)+ .select('agentTriggerDeletionStartedAt')+ .lean<{ agentTriggerDeletionStartedAt?: Date }>();+ if (user != null) {+ if (user.agentTriggerDeletionStartedAt?.getTime() === marker.fenceStartedAt.getTime()) {+ // Move a still-owned pre-commit marker behind this bounded scan so+ // it cannot starve later markers whose users are already gone.+ await UserPurge().updateOne(+ { _id: marker._id, fenceStartedAt: marker.fenceStartedAt },+ { $currentDate: { updatedAt: true } },+ );+ continue;+ }+ await UserPurge().deleteOne({ _id: marker._id, fenceStartedAt: marker.fenceStartedAt });+ continue;+ }+ await Promise.all([+ Delivery().deleteMany({ user: marker._id }),+ LaneSequence().deleteMany({ user: marker._id }),+ ]);+ const result = await UserPurge().deleteOne({+ _id: marker._id,+ fenceStartedAt: marker.fenceStartedAt,+ });+ recovered += result.deletedCount;+ }+ return recovered;+ }++ async function deleteAgentTriggerDeliveriesByUser(user: string | Types.ObjectId): Promise<void> {+ await Promise.all([Delivery().deleteMany({ user }), LaneSequence().deleteMany({ user })]);+ await UserPurge().deleteOne({ _id: user });+ }++ return {+ ensureAgentTriggerDeliveryIndexes,+ enqueueAgentTriggerDelivery,+ claimNextAgentTriggerDelivery,+ findEarlierAgentTriggerDelivery,+ releaseAgentTriggerDelivery,+ beginAgentTriggerDeliveryAttempt,+ deferAgentTriggerDeliveryAttempt,+ completeAgentTriggerDelivery,+ retryAgentTriggerDelivery,+ deadLetterAgentTriggerDelivery,+ getAgentTriggerDelivery,+ getAgentTriggerDeadLetters,+ requeueAgentTriggerDelivery,+ countActiveAgentTriggerDeliveriesByUser,+ recoverAgentTriggerLanePublications,+ reclaimInactiveAgentTriggerLanes,+ prepareAgentTriggerUserPurge,+ cancelAgentTriggerUserPurge,+ recoverAgentTriggerUserPurges,+ deleteAgentTriggerDeliveriesByUser,+ };+}diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.tsindex 1270a03c5b..9540026232 100644--- a/packages/data-schemas/src/methods/user.methods.spec.ts+++ b/packages/data-schemas/src/methods/user.methods.spec.ts@@ -2,8 +2,8 @@ import mongoose from 'mongoose';import { MongoMemoryServer } from 'mongodb-memory-server';import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';import type * as t from '~/types';+import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';import balanceSchema from '~/schema/balance';-import { createUserMethods } from './user';import userSchema from '~/schema/user';/** Mocking crypto for generateToken */@@ -628,6 +628,85 @@ describe('User Methods - Database Tests', () => {});});+ describe('agent trigger account-deletion fence', () => {+ test('blocks trigger admission until the owning deletion attempt releases it', async () => {+ const user = await User.create({+ name: 'Trigger Fence',+ email: '[email protected]',+ provider: 'local',+ });+ const userId = user._id.toString();+ const startedAt = new Date('2026-08-17T12:00:00.000Z');++ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);+ await expect(methods.beginAgentTriggerUserDeletion(userId, startedAt)).resolves.toBe(+ 'acquired',+ );+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);+ await expect(+ methods.beginAgentTriggerUserDeletion(userId, new Date(startedAt.getTime() + 1)),+ ).resolves.toBe('in_progress');+ await expect(+ methods.cancelAgentTriggerUserDeletion(userId, new Date(startedAt.getTime() + 1)),+ ).resolves.toBe(false);+ await expect(methods.cancelAgentTriggerUserDeletion(userId, startedAt)).resolves.toBe(true);+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);+ });++ test('reports a missing principal without creating a deletion fence', async () => {+ const userId = new mongoose.Types.ObjectId().toString();++ await expect(methods.beginAgentTriggerUserDeletion(userId, new Date())).resolves.toBe(+ 'missing',+ );+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);+ });++ test('requires explicit stale-fence recovery and preserves successor ownership', async () => {+ const user = await User.create({+ name: 'Stale Trigger Fence',+ email: '[email protected]',+ provider: 'local',+ });+ const userId = user._id.toString();+ const abandonedAt = new Date('2026-08-17T12:00:00.000Z');+ const takeoverAt = new Date(abandonedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1);++ await expect(methods.beginAgentTriggerUserDeletion(userId, abandonedAt)).resolves.toBe(+ 'acquired',+ );+ await expect(methods.beginAgentTriggerUserDeletion(userId, takeoverAt)).resolves.toBe(+ 'in_progress',+ );+ await expect(+ methods.recoverStaleAgentTriggerUserDeletion(+ userId,+ new Date(abandonedAt.getTime() + USER_DELETION_FENCE_STALE_MS - 1),+ ),+ ).resolves.toBe('in_progress');+ await expect(methods.recoverStaleAgentTriggerUserDeletion(userId, takeoverAt)).resolves.toBe(+ 'acquired',+ );+ await expect(methods.cancelAgentTriggerUserDeletion(userId, abandonedAt)).resolves.toBe(+ false,+ );+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);+ await expect(methods.cancelAgentTriggerUserDeletion(userId, takeoverAt)).resolves.toBe(true);+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);+ });++ test('rejects invalid deletion-fence timestamps', async () => {+ const userId = new mongoose.Types.ObjectId().toString();++ await expect(+ methods.beginAgentTriggerUserDeletion(userId, new Date(Number.NaN)),+ ).rejects.toThrow('startedAt must be a valid Date');+ await expect(+ methods.recoverStaleAgentTriggerUserDeletion(userId, new Date(Number.NaN)),+ ).rejects.toThrow('recoveredAt must be a valid Date');+ });+ });+describe('countUsers', () => {test('should count all users', async () => {await User.create([diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.tsindex d03eb7fed6..2d324bfcdc 100644--- a/packages/data-schemas/src/methods/user.ts+++ b/packages/data-schemas/src/methods/user.ts@@ -12,6 +12,8 @@ import { signPayload } from '~/crypto';/** Default JWT session expiry: 15 minutes in milliseconds */export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15;+/** Minimum age before an explicitly offline operator may recover an abandoned deletion fence. */+export const USER_DELETION_FENCE_STALE_MS: number = 15 * 60_000;interface UserMethodDeps {getCache?: (key: string) => CacheStore | undefined;@@ -117,6 +119,16 @@ export function createUserMethods(>;getUserById: (userId: string, fieldsToSelect?: string | string[] | null) => Promise<IUser | null>;generateToken: (user: IUser, expiresIn?: number) => Promise<string>;+ beginAgentTriggerUserDeletion: (+ userId: string,+ startedAt: Date,+ ) => Promise<'acquired' | 'in_progress' | 'missing'>;+ recoverStaleAgentTriggerUserDeletion: (+ userId: string,+ recoveredAt: Date,+ ) => Promise<'acquired' | 'in_progress' | 'missing'>;+ cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;+ isAgentTriggerPrincipalActive: (userId: string) => Promise<boolean>;deleteUserById: (userId: string) => Promise<UserDeleteResult>;updateUserPlugins: (userId: string,@@ -376,6 +388,70 @@ export function createUserMethods(}}+ /** Establishes the durable admission fence used while account deletion drains triggers. */+ async function beginAgentTriggerUserDeletion(+ userId: string,+ startedAt: Date,+ ): Promise<'acquired' | 'in_progress' | 'missing'> {+ if (!(startedAt instanceof Date) || !Number.isFinite(startedAt.getTime())) {+ throw new TypeError('startedAt must be a valid Date');+ }+ const User = mongoose.models.User;+ const result = await User.updateOne(+ { _id: userId, agentTriggerDeletionStartedAt: { $exists: false } },+ { $set: { agentTriggerDeletionStartedAt: startedAt } },+ );+ if (result.modifiedCount === 1) {+ await invalidateAuthUserDocCache(userId);+ return 'acquired';+ }+ return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress';+ }++ /** Replaces an abandoned fence only for an operator-confirmed offline deployment.+ * Normal request paths must never call this: age alone cannot prove the prior owner died. */+ async function recoverStaleAgentTriggerUserDeletion(+ userId: string,+ recoveredAt: Date,+ ): Promise<'acquired' | 'in_progress' | 'missing'> {+ if (!(recoveredAt instanceof Date) || !Number.isFinite(recoveredAt.getTime())) {+ throw new TypeError('recoveredAt must be a valid Date');+ }+ const User = mongoose.models.User;+ const staleBefore = new Date(recoveredAt.getTime() - USER_DELETION_FENCE_STALE_MS);+ const result = await User.updateOne(+ { _id: userId, agentTriggerDeletionStartedAt: { $lte: staleBefore } },+ { $set: { agentTriggerDeletionStartedAt: recoveredAt } },+ );+ if (result.modifiedCount === 1) {+ await invalidateAuthUserDocCache(userId);+ return 'acquired';+ }+ return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress';+ }++ /** Releases only the account-deletion attempt that owns this exact fence. */+ async function cancelAgentTriggerUserDeletion(userId: string, startedAt: Date): Promise<boolean> {+ const User = mongoose.models.User;+ const result = await User.updateOne(+ { _id: userId, agentTriggerDeletionStartedAt: startedAt },+ { $unset: { agentTriggerDeletionStartedAt: 1 } },+ );+ if (result.modifiedCount === 1) {+ await invalidateAuthUserDocCache(userId);+ return true;+ }+ return false;+ }++ async function isAgentTriggerPrincipalActive(userId: string): Promise<boolean> {+ const User = mongoose.models.User;+ return (+ (await User.exists({ _id: userId, agentTriggerDeletionStartedAt: { $exists: false } })) !=+ null+ );+ }+/*** Generates a JWT token for a given user.* @param user - The user object@@ -627,6 +703,10 @@ export function createUserMethods(searchUsers,getUserById,generateToken,+ beginAgentTriggerUserDeletion,+ recoverStaleAgentTriggerUserDeletion,+ cancelAgentTriggerUserDeletion,+ isAgentTriggerPrincipalActive,deleteUserById,updateUserPlugins,toggleUserMemories,diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.tsindex ed1e56d9a5..06d6a818f8 100644--- a/packages/data-schemas/src/models/index.ts+++ b/packages/data-schemas/src/models/index.ts@@ -1,4 +1,7 @@+import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';import { createSkillSyncCredentialModel } from './skillSyncCredential';+import { createAgentTriggerUserPurgeModel } from './triggerUserPurge';+import { createAgentTriggerDeliveryModel } from './triggerDelivery';import { createSkillSyncStatusModel } from './skillSyncStatus';import { createConversationTagModel } from './conversationTag';import { createAgentCategoryModel } from './agentCategory';@@ -78,6 +81,9 @@ export function createModels(mongoose: typeof import('mongoose')): {AuditLog: ReturnType<typeof createAuditLogModel>;Group: ReturnType<typeof createGroupModel>;Config: ReturnType<typeof createConfigModel>;+ AgentTriggerDelivery: ReturnType<typeof createAgentTriggerDeliveryModel>;+ AgentTriggerLaneSequence: ReturnType<typeof createAgentTriggerLaneSequenceModel>;+ AgentTriggerUserPurge: ReturnType<typeof createAgentTriggerUserPurgeModel>;} {const models = {User: createUserModel(mongoose),@@ -117,6 +123,9 @@ export function createModels(mongoose: typeof import('mongoose')): {AuditLog: createAuditLogModel(mongoose),Group: createGroupModel(mongoose),Config: createConfigModel(mongoose),+ AgentTriggerDelivery: createAgentTriggerDeliveryModel(mongoose),+ AgentTriggerLaneSequence: createAgentTriggerLaneSequenceModel(mongoose),+ AgentTriggerUserPurge: createAgentTriggerUserPurgeModel(mongoose),};/*** Background index builds fail silently unless an 'index' listener isdiff --git a/packages/data-schemas/src/models/triggerDelivery.ts b/packages/data-schemas/src/models/triggerDelivery.tsnew file mode 100644index 0000000000..0802429465--- /dev/null+++ b/packages/data-schemas/src/models/triggerDelivery.ts@@ -0,0 +1,14 @@+import { Model } from 'mongoose';+import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';+import triggerDeliverySchema from '~/schema/triggerDelivery';++export function createAgentTriggerDeliveryModel(+ mongoose: typeof import('mongoose'),+): Model<IAgentTriggerDeliveryDocument> {+ applyTenantIsolation(triggerDeliverySchema);+ return (+ mongoose.models.AgentTriggerDelivery ||+ mongoose.model<IAgentTriggerDeliveryDocument>('AgentTriggerDelivery', triggerDeliverySchema)+ );+}diff --git a/packages/data-schemas/src/models/triggerLaneSequence.ts b/packages/data-schemas/src/models/triggerLaneSequence.tsnew file mode 100644index 0000000000..04e4f2aca5--- /dev/null+++ b/packages/data-schemas/src/models/triggerLaneSequence.ts@@ -0,0 +1,17 @@+import { Model } from 'mongoose';+import type { IAgentTriggerLaneSequenceDocument } from '~/types/triggerDelivery';+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';+import triggerLaneSequenceSchema from '~/schema/triggerLaneSequence';++export function createAgentTriggerLaneSequenceModel(+ mongoose: typeof import('mongoose'),+): Model<IAgentTriggerLaneSequenceDocument> {+ applyTenantIsolation(triggerLaneSequenceSchema);+ return (+ mongoose.models.AgentTriggerLaneSequence ||+ mongoose.model<IAgentTriggerLaneSequenceDocument>(+ 'AgentTriggerLaneSequence',+ triggerLaneSequenceSchema,+ )+ );+}diff --git a/packages/data-schemas/src/models/triggerUserPurge.ts b/packages/data-schemas/src/models/triggerUserPurge.tsnew file mode 100644index 0000000000..3e4837c7ad--- /dev/null+++ b/packages/data-schemas/src/models/triggerUserPurge.ts@@ -0,0 +1,14 @@+import { Model } from 'mongoose';+import type { IAgentTriggerUserPurgeDocument } from '~/types/triggerDelivery';+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';+import triggerUserPurgeSchema from '~/schema/triggerUserPurge';++export function createAgentTriggerUserPurgeModel(+ mongoose: typeof import('mongoose'),+): Model<IAgentTriggerUserPurgeDocument> {+ applyTenantIsolation(triggerUserPurgeSchema);+ return (+ mongoose.models.AgentTriggerUserPurge ||+ mongoose.model<IAgentTriggerUserPurgeDocument>('AgentTriggerUserPurge', triggerUserPurgeSchema)+ );+}diff --git a/packages/data-schemas/src/schema/index.ts b/packages/data-schemas/src/schema/index.tsindex e6ac941d00..d9171d0099 100644--- a/packages/data-schemas/src/schema/index.ts+++ b/packages/data-schemas/src/schema/index.ts@@ -32,3 +32,6 @@ export { default as groupSchema } from './group';export { default as systemGrantSchema } from './systemGrant';export { default as auditLogSchema } from './auditLog';export { default as configSchema } from './config';+export { default as triggerDeliverySchema } from './triggerDelivery';+export { default as triggerLaneSequenceSchema } from './triggerLaneSequence';+export { default as triggerUserPurgeSchema } from './triggerUserPurge';diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.tsnew file mode 100644index 0000000000..11f874c6bf--- /dev/null+++ b/packages/data-schemas/src/schema/triggerDelivery.ts@@ -0,0 +1,71 @@+import { Schema } from 'mongoose';+import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';++const failureSchema = new Schema(+ {+ code: { type: String, required: true, maxlength: 128 },+ message: { type: String, required: true, maxlength: 2048 },+ certainty: { type: String, enum: ['definite', 'ambiguous'], required: true },+ retryable: { type: Boolean, required: true },+ attemptedAt: { type: Date, required: true },+ status: { type: Number },+ },+ { _id: false },+);++const historySchema = new Schema(+ {+ attempt: { type: Number, required: true, min: 1 },+ outcome: { type: String, enum: ['succeeded', 'retry', 'dead'], required: true },+ at: { type: Date, required: true },+ workerId: { type: String, required: true },+ error: { type: failureSchema },+ },+ { _id: false },+);++const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(+ {+ deliveryKey: { type: String, required: true, maxlength: 128 },+ fingerprint: { type: String, required: true, maxlength: 128 },+ orderingKey: { type: String, required: true, maxlength: 128 },+ // Zero is reserved for a staging row that is visible before sequence allocation.+ laneSequence: { type: Number, required: true, min: 0 },+ envelope: { type: Schema.Types.Mixed, required: true },+ user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true },+ tenantId: { type: String, index: true },+ status: {+ type: String,+ enum: ['staging', 'pending', 'leased', 'succeeded', 'dead'],+ required: true,+ default: 'pending',+ },+ attempts: { type: Number, required: true, default: 0, min: 0 },+ availableAt: { type: Date, required: true },+ leaseBy: { type: String },+ leaseUntil: { type: Date },+ claimToken: { type: String },+ lastError: { type: failureSchema },+ result: { type: Schema.Types.Mixed },+ history: { type: [historySchema], default: undefined },+ settledAt: { type: Date },+ expiresAt: { type: Date },+ requeueCount: { type: Number, default: 0, min: 0 },+ stagingRecoveryAt: { type: Date },+ laneCleanupPendingAt: { type: Date },+ },+ { timestamps: true },+);++triggerDeliverySchema.index({ deliveryKey: 1 }, { unique: true });+triggerDeliverySchema.index({ status: 1, availableAt: 1, createdAt: 1 });+triggerDeliverySchema.index({ status: 1, leaseUntil: 1, createdAt: 1 });+triggerDeliverySchema.index({ orderingKey: 1, status: 1, laneSequence: 1 });+triggerDeliverySchema.index({ status: 1, updatedAt: -1 });+triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });+triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });+// Only successful rows receive expiresAt. Dead letters remain available until+// an operator explicitly requeues or removes them.+triggerDeliverySchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });++export default triggerDeliverySchema;diff --git a/packages/data-schemas/src/schema/triggerLaneSequence.ts b/packages/data-schemas/src/schema/triggerLaneSequence.tsnew file mode 100644index 0000000000..5a6bbb1369--- /dev/null+++ b/packages/data-schemas/src/schema/triggerLaneSequence.ts@@ -0,0 +1,23 @@+import { Schema } from 'mongoose';+import type { IAgentTriggerLaneSequenceDocument } from '~/types/triggerDelivery';++const triggerLaneSequenceSchema: Schema<IAgentTriggerLaneSequenceDocument> = new Schema(+ {+ _id: { type: String, required: true },+ value: { type: Number, required: true, min: 1 },+ user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true },+ tenantId: { type: String, index: true },+ tailDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },+ publisherDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },+ publisherStartedAt: { type: Date },+ cleanupRequestedAt: { type: Date, index: true },+ },+ { timestamps: true },+);++// Publisher fields exist only while a lane admission is in flight. The sparse+// index keeps the periodic recovery scan proportional to abandoned publishers+// instead of all retained lane history (and is supported by DocumentDB 3.6+).+triggerLaneSequenceSchema.index({ publisherStartedAt: 1 }, { sparse: true });++export default triggerLaneSequenceSchema;diff --git a/packages/data-schemas/src/schema/triggerUserPurge.ts b/packages/data-schemas/src/schema/triggerUserPurge.tsnew file mode 100644index 0000000000..d627c5c998--- /dev/null+++ b/packages/data-schemas/src/schema/triggerUserPurge.ts@@ -0,0 +1,13 @@+import { Schema } from 'mongoose';+import type { IAgentTriggerUserPurgeDocument } from '~/types/triggerDelivery';++const triggerUserPurgeSchema: Schema<IAgentTriggerUserPurgeDocument> = new Schema(+ {+ _id: { type: Schema.Types.ObjectId, ref: 'User', required: true },+ fenceStartedAt: { type: Date, required: true },+ tenantId: { type: String, index: true },+ },+ { timestamps: true },+);++export default triggerUserPurgeSchema;diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.tsindex 309ad54aa9..79a9a64195 100644--- a/packages/data-schemas/src/schema/user.ts+++ b/packages/data-schemas/src/schema/user.ts@@ -131,6 +131,10 @@ const userSchema: Schema<IUser> = new Schema<IUser>(type: Date,default: null,},+ agentTriggerDeletionStartedAt: {+ type: Date,+ select: false,+ },personalization: {type: {memories: {diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.tsindex 7a96064f87..cecdab0a50 100644--- a/packages/data-schemas/src/types/index.ts+++ b/packages/data-schemas/src/types/index.ts@@ -29,6 +29,7 @@ export * from './prompts';/* Skills */export * from './skill';export * from './skillSync';+export * from './triggerDelivery';/* Access Control */export * from './accessRole';export * from './aclEntry';diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.tsnew file mode 100644index 0000000000..e29fd2b7e6--- /dev/null+++ b/packages/data-schemas/src/types/triggerDelivery.ts@@ -0,0 +1,105 @@+import type { Document, Types } from 'mongoose';++export type AgentTriggerDeliveryStatus = 'staging' | 'pending' | 'leased' | 'succeeded' | 'dead';+export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';++export interface AgentTriggerDeliveryFailure {+ code: string;+ message: string;+ certainty: 'definite' | 'ambiguous';+ retryable: boolean;+ attemptedAt: Date;+ status?: number;+}++export interface AgentTriggerDeliveryHistoryEntry {+ attempt: number;+ outcome: AgentTriggerDeliveryOutcome;+ at: Date;+ workerId: string;+ error?: AgentTriggerDeliveryFailure;+}++export interface IAgentTriggerDelivery {+ _id?: Types.ObjectId;+ deliveryKey: string;+ fingerprint: string;+ orderingKey: string;+ /** Monotonic sequence allocated while holding the lane publication fence. */+ laneSequence: number;+ envelope: unknown;+ user: Types.ObjectId;+ tenantId?: string;+ status: AgentTriggerDeliveryStatus;+ attempts: number;+ availableAt: Date;+ leaseBy?: string;+ leaseUntil?: Date;+ claimToken?: string;+ lastError?: AgentTriggerDeliveryFailure;+ result?: unknown;+ history?: AgentTriggerDeliveryHistoryEntry[];+ settledAt?: Date;+ expiresAt?: Date;+ requeueCount?: number;+ /** Fairness cursor for bounded recovery of rows stranded before lane publication. */+ stagingRecoveryAt?: Date;+ /** Durable proof that successful settlement still owes lane cleanup publication. */+ laneCleanupPendingAt?: Date;+ createdAt?: Date;+ updatedAt?: Date;+}++export interface IAgentTriggerDeliveryDocument+ extends Omit<IAgentTriggerDelivery, '_id'>,+ Document {}++export interface AgentTriggerDeliveryRecord+ extends Omit<IAgentTriggerDelivery, '_id' | 'createdAt'> {+ id: string;+ createdAt: Date;+}++export interface IAgentTriggerLaneSequence {+ _id: string;+ value: number;+ user: Types.ObjectId;+ tenantId?: string;+ /** Latest delivery admitted to this lane. Used to reclaim inactive lane counters safely. */+ tailDeliveryId?: Types.ObjectId;+ /** Delivery currently owning the serialized sequence/publication step. */+ publisherDeliveryId?: Types.ObjectId;+ publisherStartedAt?: Date;+ cleanupRequestedAt?: Date;+ createdAt?: Date;+ updatedAt?: Date;+}++export interface IAgentTriggerLaneSequenceDocument+ extends Omit<IAgentTriggerLaneSequence, '_id'>,+ Document<string> {}++/** Durable proof that trigger payload cleanup must survive the deleting process. */+export interface IAgentTriggerUserPurge {+ _id: Types.ObjectId;+ fenceStartedAt: Date;+ tenantId?: string;+ createdAt?: Date;+ updatedAt?: Date;+}++export interface IAgentTriggerUserPurgeDocument+ extends Omit<IAgentTriggerUserPurge, '_id'>,+ Document {}++export interface AgentTriggerDeliveryClaim extends AgentTriggerDeliveryRecord {+ claimToken: string;+ leaseBy: string;+ leaseUntil: Date;+ status: 'leased';+}++export interface AgentTriggerOrderingBlock {+ availableAt: Date;+ leaseUntil?: Date;+}diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.tsindex 2b91252a7a..169209cf4a 100644--- a/packages/data-schemas/src/types/user.ts+++ b/packages/data-schemas/src/types/user.ts@@ -53,6 +53,8 @@ export interface IUser extends Document {expiresAt?: Date;termsAccepted?: boolean;termsAcceptedAt?: Date | null;+ /** Internal fence that prevents agent-trigger admission during account deletion. */+ agentTriggerDeletionStartedAt?: Date;personalization?: {memories?: boolean;statefulCodeEnvironment?: StatefulCodeEnvironment;