Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📬 feat: Add Durable Agent Trigger Delivery (#14925)

Danny Avilaaa35cd42b149
diff --git a/.env.example b/.env.example
index 0613685b58..301ab8adb6 100644
--- a/.env.example
+++ b/.env.example
@@ -1204,6 +1204,19 @@ OPENWEATHER_API_KEY=
# or
# COHERE_API_KEY=your_cohere_api_key
+#===========================#
+# Agent Trigger Delivery #
+#===========================#
+
+# Base URL used by trusted in-process event producers to dispatch agent fires and steers.
+# Defaults to this process's bound listener. Set only when internal trigger admission must
+# traverse another trusted HTTP origin, such as a TLS front door.
+# AGENT_TRIGGERS_SELF_URL=http://127.0.0.1:3080
+
+# Trusted event adapters enqueue through the shared durable trigger service. Mongo-backed
+# leases make its workers safe across replicas; successful delivery records expire after
+# 90 days, while dead letters remain available for explicit operator requeue.
+
#======================#
# MCP Configuration #
#======================#
diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js
index 8cc32e4a85..b8fecb4f8e 100644
--- a/api/server/controllers/UserController.js
+++ b/api/server/controllers/UserController.js
@@ -3,6 +3,7 @@ const { logger, getTenantId, webSearchKeys } = require('@librechat/data-schemas'
const {
getNewS3URL,
needsRefresh,
+ GenerationJobManager,
MCPOAuthHandler,
MCPTokenStorage,
getAppConfigOptionsFromUser,
@@ -24,6 +25,12 @@ const { verifyEmail, resendVerificationEmail } = require('~/server/services/Auth
const { getMCPManager, getFlowStateManager, getMCPServersRegistry } = require('~/config');
const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');
const { processDeleteRequest } = require('~/server/services/Files/process');
+const {
+ drainAgentTriggerDeliveriesForUser,
+ prepareAgentTriggerUserPurge,
+ cancelAgentTriggerUserPurge,
+ purgeAgentTriggerDeliveriesForUser,
+} = require('~/server/services/Agents/triggers');
const { getAppConfig } = require('~/server/services/Config');
const { getLogStores } = require('~/cache');
const db = require('~/models');
@@ -351,6 +358,8 @@ const updateUserPluginsController = async (req, res) => {
const deleteUserController = async (req, res) => {
const { user } = req;
+ let triggerDeletionFence;
+ let userDeleted = false;
try {
const existingUser = await db.getUserById(
@@ -369,6 +378,31 @@ const deleteUserController = async (req, res) => {
}
}
+ // Block new trigger admissions across replicas while preserving the user
+ // principal so a transient cleanup failure remains retryable.
+ triggerDeletionFence = new Date();
+ const fenceState = await db.beginAgentTriggerUserDeletion(user.id, triggerDeletionFence);
+ if (fenceState === 'in_progress') {
+ triggerDeletionFence = undefined;
+ throw new Error('Agent trigger account deletion is already in progress');
+ }
+ if (fenceState === 'missing') {
+ triggerDeletionFence = undefined;
+ }
+ if (triggerDeletionFence != null) {
+ await prepareAgentTriggerUserPurge(user.id, triggerDeletionFence, user.tenantId);
+ }
+ await drainAgentTriggerDeliveriesForUser(user.id);
+ const activeAgentRuns = await GenerationJobManager.getCleanupBlockingJobIdsForUser(
+ user.id,
+ user.tenantId,
+ );
+ await Promise.all(
+ activeAgentRuns.map((streamId) =>
+ GenerationJobManager.abortJob(streamId, { awaitProviderDrain: true }),
+ ),
+ );
+
await db.deleteMessages({ user: user.id });
await db.deleteAllUserSessions({ userId: user.id });
await db.deleteTransactions({ user: user.id });
@@ -394,7 +428,6 @@ const deleteUserController = async (req, res) => {
logger.error('[deleteUserController] Error deleting user convos, likely no convos', error);
}
await deleteUserPluginAuth(user.id, null, true);
- await db.deleteUserById(user.id);
await deleteAllSharedLinksWithCleanup(user.id);
await deleteUserFiles(req);
await db.deleteFiles(null, user.id);
@@ -411,9 +444,30 @@ const deleteUserController = async (req, res) => {
await db.deleteTokens({ userId: user.id });
await db.removeUserFromAllGroups(user.id);
await db.deleteAclEntries({ principalId: user._id });
+ const deleteResult = await db.deleteUserById(user.id);
+ if (deleteResult.deletedCount !== 1) {
+ throw new Error('User disappeared before account deletion could commit');
+ }
+ userDeleted = true;
+ await purgeAgentTriggerDeliveriesForUser(user.id);
logger.info(`User deleted account. Email: ${user.email} ID: ${user.id}`);
res.status(200).send({ message: 'User deleted' });
} catch (err) {
+ if (triggerDeletionFence != null && !userDeleted) {
+ try {
+ await cancelAgentTriggerUserPurge(user.id, triggerDeletionFence);
+ } catch (purgeFenceError) {
+ logger.error(
+ '[deleteUserController] Failed to disarm trigger purge recovery',
+ purgeFenceError,
+ );
+ }
+ try {
+ await db.cancelAgentTriggerUserDeletion(user.id, triggerDeletionFence);
+ } catch (fenceError) {
+ logger.error('[deleteUserController] Failed to release trigger deletion fence', fenceError);
+ }
+ }
logger.error('[deleteUserController]', err);
return res.status(500).json({ message: 'Something went wrong.' });
}
diff --git a/api/server/controllers/UserController.spec.js b/api/server/controllers/UserController.spec.js
index 1466dce291..b21fe904e3 100644
--- a/api/server/controllers/UserController.spec.js
+++ b/api/server/controllers/UserController.spec.js
@@ -1,6 +1,13 @@
const mongoose = require('mongoose');
const { MongoMemoryServer } = require('mongodb-memory-server');
+const mockGetActiveJobIdsForUser = jest.fn().mockResolvedValue([]);
+const mockAbortJob = jest.fn().mockResolvedValue({ success: true });
+const mockDrainAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undefined);
+const mockPrepareAgentTriggerUserPurge = jest.fn().mockResolvedValue(undefined);
+const mockCancelAgentTriggerUserPurge = jest.fn().mockResolvedValue(true);
+const mockPurgeAgentTriggerDeliveriesForUser = jest.fn().mockResolvedValue(undefined);
+
jest.mock('@librechat/data-schemas', () => {
const actual = jest.requireActual('@librechat/data-schemas');
return {
@@ -26,7 +33,9 @@ jest.mock('~/models', () => {
deleteAclEntries: jest.fn().mockResolvedValue(undefined),
updateUserPlugins: jest.fn(),
deleteAssistants: jest.fn().mockResolvedValue(undefined),
- deleteUserById: jest.fn().mockResolvedValue(undefined),
+ deleteUserById: jest.fn().mockResolvedValue({ deletedCount: 1 }),
+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),
+ cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),
deleteUserPrompts: jest.fn().mockResolvedValue(undefined),
deleteUserSkills: jest.fn().mockResolvedValue(undefined),
deleteMessages: jest.fn().mockResolvedValue(undefined),
@@ -73,6 +82,17 @@ jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
needsRefresh: jest.fn(),
getNewS3URL: jest.fn(),
+ GenerationJobManager: {
+ getCleanupBlockingJobIdsForUser: (...args) => mockGetActiveJobIdsForUser(...args),
+ abortJob: (...args) => mockAbortJob(...args),
+ },
+}));
+
+jest.mock('~/server/services/Agents/triggers', () => ({
+ drainAgentTriggerDeliveriesForUser: (...args) => mockDrainAgentTriggerDeliveriesForUser(...args),
+ prepareAgentTriggerUserPurge: (...args) => mockPrepareAgentTriggerUserPurge(...args),
+ cancelAgentTriggerUserPurge: (...args) => mockCancelAgentTriggerUserPurge(...args),
+ purgeAgentTriggerDeliveriesForUser: (...args) => mockPurgeAgentTriggerDeliveriesForUser(...args),
}));
jest.mock('~/server/services/Files/process', () => ({
@@ -117,7 +137,14 @@ const {
verifyEmailController,
} = require('./UserController');
const { Group } = require('~/db/models');
-const { deleteConvos, acceptTerms } = require('~/models');
+const {
+ deleteConvos,
+ acceptTerms,
+ deleteUserById,
+ deleteMessages,
+ beginAgentTriggerUserDeletion,
+ cancelAgentTriggerUserDeletion,
+} = require('~/models');
const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService');
describe('verifyEmailController', () => {
@@ -321,6 +348,77 @@ describe('deleteUserController', () => {
expect(mockRes.status).toHaveBeenCalledWith(200);
expect(mockRes.send).toHaveBeenCalledWith({ message: 'User deleted' });
+ expect(beginAgentTriggerUserDeletion).toHaveBeenCalledWith(userId.toString(), expect.any(Date));
+ expect(mockPrepareAgentTriggerUserPurge).toHaveBeenCalledWith(
+ userId.toString(),
+ expect.any(Date),
+ undefined,
+ );
+ expect(mockDrainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());
+ expect(beginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(
+ mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0],
+ );
+ expect(mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0]).toBeLessThan(
+ mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0],
+ );
+ expect(mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0]).toBeLessThan(
+ deleteMessages.mock.invocationCallOrder[0],
+ );
+ expect(deleteMessages.mock.invocationCallOrder[0]).toBeLessThan(
+ deleteUserById.mock.invocationCallOrder[0],
+ );
+ expect(deleteUserById.mock.invocationCallOrder[0]).toBeLessThan(
+ mockPurgeAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0],
+ );
+ expect(mockPurgeAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(userId.toString());
+ expect(cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
+ expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();
+ });
+
+ it('aborts generations admitted before the deletion fence before erasing messages', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ mockGetActiveJobIdsForUser.mockResolvedValueOnce(['stream-1', 'stream-2']);
+ const req = {
+ user: {
+ id: userId.toString(),
+ _id: userId,
+ email: '[email protected]',
+ tenantId: 'tenant-1',
+ },
+ };
+
+ await deleteUserController(req, mockRes);
+
+ expect(mockGetActiveJobIdsForUser).toHaveBeenCalledWith(userId.toString(), 'tenant-1');
+ expect(mockAbortJob).toHaveBeenCalledWith('stream-1', { awaitProviderDrain: true });
+ expect(mockAbortJob).toHaveBeenCalledWith('stream-2', { awaitProviderDrain: true });
+ expect(mockAbortJob.mock.invocationCallOrder[1]).toBeLessThan(
+ deleteMessages.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('fails closed and releases deletion fences when a provider cannot confirm drain', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ const userIdString = userId.toString();
+ mockGetActiveJobIdsForUser.mockResolvedValueOnce(['stream-still-writing']);
+ mockAbortJob.mockRejectedValueOnce(new Error('provider drain timed out'));
+ const req = {
+ user: {
+ id: userIdString,
+ _id: userId,
+ email: '[email protected]',
+ tenantId: 'tenant-1',
+ },
+ };
+
+ await deleteUserController(req, mockRes);
+
+ expect(mockRes.status).toHaveBeenCalledWith(500);
+ expect(deleteMessages).not.toHaveBeenCalled();
+ const deletionFence = beginAgentTriggerUserDeletion.mock.calls[0][1];
+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(userIdString, deletionFence);
+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(userIdString, deletionFence);
+ expect(deleteUserById).not.toHaveBeenCalled();
});
it('should remove the user from all groups via $pullAll', async () => {
@@ -393,6 +491,38 @@ describe('deleteUserController', () => {
expect(mockRes.status).toHaveBeenCalledWith(500);
expect(mockRes.json).toHaveBeenCalledWith({ message: 'Something went wrong.' });
+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
+ userId.toString(),
+ expect.any(Date),
+ );
+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(
+ userId.toString(),
+ expect.any(Date),
+ );
+ expect(deleteUserById).not.toHaveBeenCalled();
+ });
+
+ it('preserves queued trigger payloads when deletion fails before the user commit', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ deleteMessages.mockRejectedValueOnce(new Error('db down'));
+
+ await deleteUserController({ user: { id: String(userId), _id: userId } }, mockRes);
+
+ expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));
+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));
+ });
+
+ it('does not purge trigger payloads when the user deletion did not commit', async () => {
+ const userId = new mongoose.Types.ObjectId();
+ deleteUserById.mockResolvedValueOnce({ deletedCount: 0 });
+
+ await deleteUserController({ user: { id: String(userId), _id: userId } }, mockRes);
+
+ expect(mockPurgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
+ expect(mockCancelAgentTriggerUserPurge).toHaveBeenCalledWith(String(userId), expect.any(Date));
+ expect(cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(String(userId), expect.any(Date));
+ expect(mockRes.status).toHaveBeenCalledWith(500);
});
it('should use string user.id (not ObjectId user._id) for memberIds removal', async () => {
diff --git a/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js b/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js
index 3a4b365646..545283ce26 100644
--- a/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js
+++ b/api/server/controllers/__tests__/UserController.mcpOAuth.spec.js
@@ -90,6 +90,13 @@ jest.mock('~/server/services/Files/process', () => ({
processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),
}));
+jest.mock('~/server/services/Agents/triggers', () => ({
+ drainAgentTriggerDeliveriesForUser: jest.fn(),
+ prepareAgentTriggerUserPurge: jest.fn(),
+ cancelAgentTriggerUserPurge: jest.fn(),
+ purgeAgentTriggerDeliveriesForUser: jest.fn(),
+}));
+
jest.mock('~/server/services/Config', () => ({
getAppConfig: (...args) => mockGetAppConfig(...args),
}));
diff --git a/api/server/controllers/__tests__/deleteUser.spec.js b/api/server/controllers/__tests__/deleteUser.spec.js
index 6198122bd0..9af915b772 100644
--- a/api/server/controllers/__tests__/deleteUser.spec.js
+++ b/api/server/controllers/__tests__/deleteUser.spec.js
@@ -19,6 +19,14 @@ const mockDeleteToolCalls = jest.fn();
const mockDeleteUserAgents = jest.fn();
const mockDeleteUserPrompts = jest.fn();
const mockDeleteUserSkills = jest.fn();
+const mockGetCleanupBlockingJobIdsForUser = jest.fn();
+const mockAbortJob = jest.fn();
+const mockDrainAgentTriggerDeliveriesForUser = jest.fn();
+const mockPrepareAgentTriggerUserPurge = jest.fn();
+const mockCancelAgentTriggerUserPurge = jest.fn();
+const mockPurgeAgentTriggerDeliveriesForUser = jest.fn();
+const mockBeginAgentTriggerUserDeletion = jest.fn();
+const mockCancelAgentTriggerUserDeletion = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
logger: { error: jest.fn(), info: jest.fn() },
@@ -40,6 +48,10 @@ jest.mock('@librechat/api', () => ({
needsRefresh: jest.fn(),
getNewS3URL: jest.fn(),
deleteAllSharedLinksWithCleanup: (...args) => mockDeleteAllSharedLinksWithCleanup(...args),
+ GenerationJobManager: {
+ getCleanupBlockingJobIdsForUser: (...args) => mockGetCleanupBlockingJobIdsForUser(...args),
+ abortJob: (...args) => mockAbortJob(...args),
+ },
}));
jest.mock('~/models', () => ({
@@ -47,6 +59,8 @@ jest.mock('~/models', () => ({
deleteAllSharedLinks: (...args) => mockDeleteAllSharedLinks(...args),
updateUserPlugins: (...args) => mockUpdateUserPlugins(...args),
deleteUserById: (...args) => mockDeleteUserById(...args),
+ beginAgentTriggerUserDeletion: (...args) => mockBeginAgentTriggerUserDeletion(...args),
+ cancelAgentTriggerUserDeletion: (...args) => mockCancelAgentTriggerUserDeletion(...args),
deleteMessages: (...args) => mockDeleteMessages(...args),
deletePresets: (...args) => mockDeletePresets(...args),
deleteUserKey: (...args) => mockDeleteUserKey(...args),
@@ -101,6 +115,13 @@ jest.mock('~/server/services/Files/process', () => ({
processDeleteRequest: (...args) => mockProcessDeleteRequest(...args),
}));
+jest.mock('~/server/services/Agents/triggers', () => ({
+ drainAgentTriggerDeliveriesForUser: (...args) => mockDrainAgentTriggerDeliveriesForUser(...args),
+ prepareAgentTriggerUserPurge: (...args) => mockPrepareAgentTriggerUserPurge(...args),
+ cancelAgentTriggerUserPurge: (...args) => mockCancelAgentTriggerUserPurge(...args),
+ purgeAgentTriggerDeliveriesForUser: (...args) => mockPurgeAgentTriggerDeliveriesForUser(...args),
+}));
+
jest.mock('~/server/services/Config', () => ({
getAppConfig: jest.fn(),
}));
@@ -126,7 +147,7 @@ function stubDeletionMocks() {
mockDeletePresets.mockResolvedValue();
mockDeleteConvos.mockResolvedValue();
mockDeleteUserPluginAuth.mockResolvedValue();
- mockDeleteUserById.mockResolvedValue();
+ mockDeleteUserById.mockResolvedValue({ deletedCount: 1 });
mockDeleteAllSharedLinks.mockResolvedValue();
mockDeleteAllSharedLinksWithCleanup.mockResolvedValue({ deletedCount: 0 });
mockGetFiles.mockResolvedValue([]);
@@ -136,6 +157,14 @@ function stubDeletionMocks() {
mockDeleteUserAgents.mockResolvedValue();
mockDeleteUserPrompts.mockResolvedValue();
mockDeleteUserSkills.mockResolvedValue(0);
+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValue([]);
+ mockAbortJob.mockResolvedValue({ success: true });
+ mockDrainAgentTriggerDeliveriesForUser.mockResolvedValue();
+ mockPrepareAgentTriggerUserPurge.mockResolvedValue();
+ mockCancelAgentTriggerUserPurge.mockResolvedValue(true);
+ mockPurgeAgentTriggerDeliveriesForUser.mockResolvedValue();
+ mockBeginAgentTriggerUserDeletion.mockResolvedValue('acquired');
+ mockCancelAgentTriggerUserDeletion.mockResolvedValue(true);
}
beforeEach(() => {
@@ -158,12 +187,45 @@ describe('deleteUserController - 2FA enforcement', () => {
expect(mockDeleteUserPrompts).toHaveBeenCalledWith('user1');
expect(mockDeleteUserSkills).toHaveBeenCalledWith('user1');
expect(mockVerifyOTPOrBackupCode).not.toHaveBeenCalled();
+ expect(mockBeginAgentTriggerUserDeletion.mock.invocationCallOrder[0]).toBeLessThan(
+ mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0],
+ );
+ expect(mockPrepareAgentTriggerUserPurge.mock.invocationCallOrder[0]).toBeLessThan(
+ mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0],
+ );
+ expect(mockDrainAgentTriggerDeliveriesForUser.mock.invocationCallOrder[0]).toBeLessThan(
+ mockDeleteMessages.mock.invocationCallOrder[0],
+ );
+ expect(mockDeleteMessages.mock.invocationCallOrder[0]).toBeLessThan(
+ mockDeleteUserById.mock.invocationCallOrder[0],
+ );
+ expect(mockCancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
+ expect(mockCancelAgentTriggerUserPurge).not.toHaveBeenCalled();
+ });
+
+ it('aborts active generation jobs before deleting account-owned records', async () => {
+ const req = {
+ user: { id: 'user1', _id: 'user1', email: '[email protected]', tenantId: 'tenant-1' },
+ body: {},
+ };
+ const res = createRes();
+ mockGetUserById.mockResolvedValue({ _id: 'user1', twoFactorEnabled: false });
+ mockGetCleanupBlockingJobIdsForUser.mockResolvedValueOnce(['stream-1']);
+
+ await deleteUserController(req, res);
+
+ expect(mockGetCleanupBlockingJobIdsForUser).toHaveBeenCalledWith('user1', 'tenant-1');
+ expect(mockAbortJob).toHaveBeenCalledWith('stream-1', { awaitProviderDrain: true });
+ expect(mockAbortJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockDeleteMessages.mock.invocationCallOrder[0],
+ );
});
it('proceeds with deletion when user has no 2FA record', async () => {
const req = { user: { id: 'user1', _id: 'user1', email: '[email protected]' }, body: {} };
const res = createRes();
mockGetUserById.mockResolvedValue(null);
+ mockBeginAgentTriggerUserDeletion.mockResolvedValueOnce('missing');
await deleteUserController(req, res);
diff --git a/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js b/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js
index fb46382ff6..30d687bf64 100644
--- a/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js
+++ b/api/server/controllers/__tests__/maybeUninstallOAuthMCP.spec.js
@@ -101,6 +101,13 @@ jest.mock('~/server/services/Files/process', () => ({
processDeleteRequest: jest.fn().mockResolvedValue({ deletedFileIds: [], failedFileIds: [] }),
}));
+jest.mock('~/server/services/Agents/triggers', () => ({
+ drainAgentTriggerDeliveriesForUser: jest.fn(),
+ prepareAgentTriggerUserPurge: jest.fn(),
+ cancelAgentTriggerUserPurge: jest.fn(),
+ purgeAgentTriggerDeliveriesForUser: jest.fn(),
+}));
+
jest.mock('~/server/services/Config', () => ({
getAppConfig: jest.fn(),
}));
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
index bbdd026152..c2f1173257 100644
--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
@@ -17,6 +17,7 @@ const mockTenantStorageRun = jest.fn(async (context, callback) => {
const mockSaveMessage = jest.fn();
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
+const mockIsAgentTriggerPrincipalActive = jest.fn();
const mockFilterPersistableAbortContent = jest.fn((content) => content);
const mockCheckAndIncrementPendingRequest = jest.fn();
const mockDecrementPendingRequest = jest.fn();
@@ -24,6 +25,8 @@ const mockGenerationJobManager = {
createJob: jest.fn(),
emitError: jest.fn(),
completeJob: jest.fn(),
+ beginProviderExecution: jest.fn(),
+ markProviderExecutionDrained: jest.fn(),
getResumeState: jest.fn(),
updateMetadata: jest.fn(),
claimGeneration: jest.fn(),
@@ -100,6 +103,7 @@ jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
getMessages: (...args) => mockGetMessages(...args),
getConvo: (...args) => mockGetConvo(...args),
+ isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
}));
const AgentController = require('../request');
@@ -112,9 +116,12 @@ describe('ResumableAgentController tenant context', () => {
mockDecrementPendingRequest.mockResolvedValue(undefined);
mockGetConvo.mockResolvedValue({ createdAt: '2026-07-31T00:00:00.000Z' });
mockGetMessages.mockResolvedValue([]);
+ mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);
mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
+ mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);
mockGenerationJobManager.hasJob.mockResolvedValue(true);
@@ -130,6 +137,10 @@ describe('ResumableAgentController tenant context', () => {
let allSubscribersLeftHandler;
mockGenerationJobManager.createJob.mockResolvedValue({
createdAt: 1000,
+ metadata: {
+ providerExecutionId: 'provider-segment-1',
+ providerDrained: true,
+ },
readyPromise: Promise.resolve(),
abortController: new AbortController(),
emitter: {
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index ceb1abb3ac..2ddc02abac 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -17,6 +17,8 @@ const mockGenerationJobManager = {
publishTerminalClaim: jest.fn(),
finishTerminalJob: jest.fn(),
completeJob: jest.fn(),
+ beginProviderExecution: jest.fn(),
+ markProviderExecutionDrained: jest.fn(),
failPausePersistence: jest.fn(),
getResumeState: jest.fn(),
updateMetadata: jest.fn(),
@@ -61,6 +63,7 @@ const mockFilterPersistableAbortContent = jest.fn((content) =>
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockSaveMessage = jest.fn();
+const mockIsAgentTriggerPrincipalActive = jest.fn();
const mockDeleteAgentCheckpoint = jest.fn();
const mockStartupTelemetry = {
mark: jest.fn(),
@@ -208,6 +211,7 @@ jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
getMessages: (...args) => mockGetMessages(...args),
getConvo: (...args) => mockGetConvo(...args),
+ isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
}));
const AgentController = require('../request');
@@ -245,9 +249,14 @@ describe('ResumableAgentController resume metadata', () => {
mockDecrementPendingRequest.mockResolvedValue(undefined);
mockGetConvo.mockResolvedValue({ createdAt: '2026-06-07T00:00:00.000Z' });
mockGetMessages.mockResolvedValue([]);
+ mockIsAgentTriggerPrincipalActive.mockResolvedValue(true);
mockGenerationJobManager.createJob.mockResolvedValue({
createdAt: 1000,
- metadata: { checkpointNamespace: '1000' },
+ metadata: {
+ checkpointNamespace: '1000',
+ providerExecutionId: 'provider-segment-1',
+ providerDrained: true,
+ },
readyPromise: Promise.resolve(),
abortController: new AbortController(),
emitter: { on: jest.fn() },
@@ -278,6 +287,8 @@ describe('ResumableAgentController resume metadata', () => {
);
mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
+ mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
mockGenerationJobManager.resumeClaimedGeneration.mockResolvedValue(null);
@@ -621,6 +632,91 @@ describe('ResumableAgentController resume metadata', () => {
});
});
+ it('rejects any session fenced after authentication before generation execution starts', async () => {
+ mockIsAgentTriggerPrincipalActive.mockResolvedValue(false);
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Run after a slow trigger admission.',
+ messageId: 'user-message',
+ clientRequestId: 'trigger-request',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+ const initializeClient = jest.fn();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledTimes(1);
+ expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');
+ expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0],
+ );
+ expect(mockGenerationJobManager.beginProviderExecution).not.toHaveBeenCalled();
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({
+ status: 409,
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ error: 'Account deletion is in progress',
+ generationProtocolVersion: 1,
+ });
+ expect(initializeClient).not.toHaveBeenCalled();
+ expect(mockAcceptAgentStartupTelemetry).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ JSON.stringify({
+ status: 409,
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ error: 'Account deletion is in progress',
+ }),
+ 1000,
+ );
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'trigger-request',
+ 'conversation-123',
+ DEFAULT_OWNED_CLAIM,
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ });
+
+ it('does not start a provider when account deletion or replacement wins the startup CAS', async () => {
+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(false);
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Race destructive cleanup.',
+ messageId: 'user-message',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+ const initializeClient = jest.fn();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockIsAgentTriggerPrincipalActive).toHaveBeenCalledWith('user-123');
+ expect(mockIsAgentTriggerPrincipalActive.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],
+ );
+ expect(initializeClient).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ expect.stringContaining('Generation stopped before provider startup'),
+ 1000,
+ );
+ expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(
+ 'conversation-123',
+ 1000,
+ 'provider-segment-1',
+ );
+ });
+
it('prefetches conversation state before admission and joins it with job metadata', async () => {
let resolveConversation;
let signalMetadataStarted;
@@ -3208,7 +3304,7 @@ describe('ResumableAgentController resume metadata', () => {
getHaltReason: () => 'preempt_incomplete',
};
- const runFirstTurn = async ({ run } = {}) => {
+ const runFirstTurn = async ({ run, addTitle: suppliedAddTitle } = {}) => {
let signalFinished;
const finished = new Promise((resolve) => {
signalFinished = resolve;
@@ -3216,9 +3312,11 @@ describe('ResumableAgentController resume metadata', () => {
mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());
let titleSignal;
- const addTitle = jest.fn(async (_req, options) => {
- titleSignal = options?.signal;
- });
+ const addTitle =
+ suppliedAddTitle ??
+ jest.fn(async (_req, options) => {
+ titleSignal = options?.signal;
+ });
const client = {
options: {},
@@ -3290,6 +3388,30 @@ describe('ResumableAgentController resume metadata', () => {
);
});
+ it('does not acknowledge provider drain until deferred title persistence settles', async () => {
+ resolveTitleTiming.mockReturnValueOnce('final');
+ let resolveTitle;
+ const titlePending = new Promise((resolve) => {
+ resolveTitle = resolve;
+ });
+ const addTitle = jest.fn(() => titlePending);
+
+ await runFirstTurn({ addTitle });
+
+ expect(addTitle).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.markProviderExecutionDrained).not.toHaveBeenCalled();
+
+ resolveTitle();
+ await nextTick();
+ await nextTick();
+
+ expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(
+ expect.any(String),
+ 1000,
+ 'provider-segment-1',
+ );
+ });
+
it('cancels an in-flight immediate title when the turn ends preempt-incomplete', async () => {
const { addTitle, getTitleSignal } = await runFirstTurn({ run: preemptIncompleteRun });
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js
index 0cfb9f13d1..eb30808054 100644
--- a/api/server/controllers/agents/__tests__/resume.spec.js
+++ b/api/server/controllers/agents/__tests__/resume.spec.js
@@ -60,6 +60,8 @@ const mockGenerationJobManager = {
publishTerminalClaim: jest.fn(),
finishTerminalJob: jest.fn(),
completeJob: jest.fn(),
+ beginProviderExecution: jest.fn(),
+ markProviderExecutionDrained: jest.fn(),
failPausePersistence: jest.fn(),
expireApproval: jest.fn(),
approvals: {
@@ -246,6 +248,8 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
);
mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(true);
+ mockGenerationJobManager.beginProviderExecution.mockResolvedValue(true);
+ mockGenerationJobManager.markProviderExecutionDrained.mockResolvedValue(true);
mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);
mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);
@@ -365,6 +369,32 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
// ...then torn down exactly once in the finally.
expect(mockCleanupMCPRequestContextForReq).toHaveBeenCalledTimes(1);
});
+
+ it('marks the exact resumed provider segment drained only after request cleanup', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ const resumePatch = mockGenerationJobManager.approvals.resolve.mock.calls[0][2];
+ expect(mockGenerationJobManager.beginProviderExecution).toHaveBeenCalledWith(
+ CONVO_ID,
+ 1000,
+ resumePatch.providerExecutionId,
+ );
+ expect(mockGenerationJobManager.markProviderExecutionDrained).toHaveBeenCalledWith(
+ CONVO_ID,
+ 1000,
+ resumePatch.providerExecutionId,
+ );
+ expect(mockCleanupMCPRequestContextForReq.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.markProviderExecutionDrained.mock.invocationCallOrder[0],
+ );
+ expect(
+ mockGenerationJobManager.beginProviderExecution.mock.invocationCallOrder[0],
+ ).toBeLessThan(mockInitializeClient.mock.invocationCallOrder[0]);
+ });
});
describe('request guards (rejected before claiming the action)', () => {
@@ -594,7 +624,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- { preemptCapable: true },
+ expect.objectContaining({
+ preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
+ }),
1000,
);
await settled;
@@ -687,7 +721,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- { preemptCapable: true },
+ expect.objectContaining({
+ preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
+ }),
1000,
);
await settled;
@@ -726,7 +764,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- { preemptCapable: true },
+ expect.objectContaining({
+ preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
+ }),
1000,
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
@@ -747,7 +789,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- { preemptCapable: true },
+ expect.objectContaining({
+ preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
+ }),
1000,
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
@@ -784,7 +830,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- { preemptCapable: true },
+ expect.objectContaining({
+ preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
+ }),
1000,
);
expect(mockCaptureAgentCheckpointGeneration.mock.invocationCallOrder[0]).toBeLessThan(
@@ -924,7 +974,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- { preemptCapable: true },
+ expect.objectContaining({
+ preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
+ }),
1000,
);
});
@@ -1408,8 +1462,10 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- {
+ expect.objectContaining({
preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
resolvedAskUserQuestions: [
{
request: 'What should I name the file?',
@@ -1417,7 +1473,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
contentIndex: 1,
},
],
- },
+ }),
1000,
);
expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
@@ -1449,8 +1505,10 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- {
+ expect.objectContaining({
preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
resolvedAskUserQuestions: [
{
request: 'What should I name the file?',
@@ -1458,7 +1516,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
contentMissing: true,
},
],
- },
+ }),
1000,
);
});
@@ -1484,8 +1542,10 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
CONVO_ID,
ACTION_ID,
- {
+ expect.objectContaining({
preemptCapable: true,
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
resolvedAskUserQuestions: [
{
request: { questions: expect.any(Array) },
@@ -1493,7 +1553,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
toolCallId: 'tc1',
},
],
- },
+ }),
1000,
);
});
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index ce1bdf5db9..60f3bcf9de 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -33,7 +33,7 @@ const {
cleanupMCPRequestContextForReq,
} = require('~/server/services/MCPRequestContext');
const { logViolation } = require('~/cache');
-const { saveMessage, getMessages, getConvo } = require('~/models');
+const { saveMessage, getMessages, getConvo, isAgentTriggerPrincipalActive } = require('~/models');
const {
GENERATION_PROTOCOL_HEADER,
GENERATION_PROTOCOL_V2,
@@ -858,6 +858,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
let client = null;
let jobCreatedAt;
+ let providerExecutionId;
try {
logger.debug(`[ResumableAgentController] Creating job`, {
@@ -901,10 +902,38 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
},
});
startupTelemetry?.mark('job_created');
- acceptAgentStartupTelemetry(req, streamId);
- startupTelemetry?.mark('metadata_persisted');
generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
jobCreatedAt = job.createdAt; // Capture creation time to detect job replacement
+ providerExecutionId = job.metadata?.providerExecutionId;
+
+ /** Authentication can precede a slow admission path. Recheck the durable
+ * account-deletion fence after the job is committed but before execution
+ * starts. This ordering closes both sides of the race for ordinary and
+ * trigger-scoped sessions: a fence that wins first rejects this run; a
+ * fence that starts after this read must observe the already-created job
+ * in account deletion's active-generation drain. */
+ if (!(await isAgentTriggerPrincipalActive(userId))) {
+ throw Object.assign(new Error('Account deletion is in progress'), {
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ status: 409,
+ });
+ }
+ if (
+ providerExecutionId &&
+ !(await GenerationJobManager.beginProviderExecution(
+ streamId,
+ jobCreatedAt,
+ providerExecutionId,
+ ))
+ ) {
+ throw Object.assign(new Error('Generation stopped before provider startup'), {
+ code: 'RUN_REPLACED',
+ status: 409,
+ });
+ }
+
+ acceptAgentStartupTelemetry(req, streamId);
+ startupTelemetry?.mark('metadata_persisted');
req._resumableStreamId = streamId;
getMCPRequestContext(req, undefined, { cleanupOnResponse: false });
let recoveredSteerCommitted = false;
@@ -1068,6 +1097,18 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
disposeClient(client);
}
client = null;
+ if (providerExecutionId) {
+ await GenerationJobManager.markProviderExecutionDrained?.(
+ streamId,
+ jobCreatedAt,
+ providerExecutionId,
+ ).catch((drainError) => {
+ logger.warn(
+ '[ResumableAgentController] Failed to record initialization-abort provider drain',
+ drainError,
+ );
+ });
+ }
}
return;
}
@@ -1109,6 +1150,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
};
let immediateTitlePromise = null;
+ let trailingWritePromise = null;
let backgroundClientCleanupScheduled = false;
let terminalClaim = null;
let terminalClaimFinished = false;
@@ -1720,7 +1762,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
disposeClient(client);
}
} else if (shouldGenerateTitle) {
- addTitle(req, {
+ trailingWritePromise = addTitle(req, {
text: text || getAttachmentTitleText(req.body.files),
response: { ...response },
client,
@@ -1816,27 +1858,44 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
};
// Start generation and handle any unhandled errors
- startGeneration().catch(async (err) => {
- logger.error(
- `[ResumableAgentController] Unhandled error in background generation: ${err.message}`,
- );
- startupTelemetry?.end('error', err);
- if (!pausePersistenceFailed) {
- await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
- (completeErr) => {
- logger.warn(
- '[ResumableAgentController] completeJob failed during background-error cleanup',
- completeErr,
- );
- },
+ void startGeneration()
+ .catch(async (err) => {
+ logger.error(
+ `[ResumableAgentController] Unhandled error in background generation: ${err.message}`,
);
- }
- try {
- await finishResumableRequest(req, userId);
- } finally {
- disposeBackgroundClient();
- }
- });
+ startupTelemetry?.end('error', err);
+ if (!pausePersistenceFailed) {
+ await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
+ (completeErr) => {
+ logger.warn(
+ '[ResumableAgentController] completeJob failed during background-error cleanup',
+ completeErr,
+ );
+ },
+ );
+ }
+ try {
+ await finishResumableRequest(req, userId);
+ } finally {
+ disposeBackgroundClient();
+ }
+ })
+ .finally(async () => {
+ await Promise.allSettled([immediateTitlePromise, trailingWritePromise].filter(Boolean));
+ if (providerExecutionId) {
+ await GenerationJobManager.markProviderExecutionDrained?.(
+ streamId,
+ jobCreatedAt,
+ providerExecutionId,
+ );
+ }
+ })
+ .catch((drainError) => {
+ logger.warn(
+ '[ResumableAgentController] Failed to record completed provider drain',
+ drainError,
+ );
+ });
} catch (error) {
logger.error('[ResumableAgentController] Initialization error:', error);
const initializationFailure = getInitializationFailure(error);
@@ -1939,6 +1998,18 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
if (client) {
disposeClient(client);
}
+ if (jobCreatedAt != null && providerExecutionId) {
+ await GenerationJobManager.markProviderExecutionDrained?.(
+ streamId,
+ jobCreatedAt,
+ providerExecutionId,
+ ).catch((drainError) => {
+ logger.warn(
+ '[ResumableAgentController] Failed to record initialization-error provider drain',
+ drainError,
+ );
+ });
+ }
}
};
diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js
index a5bae98f76..cb3ea82908 100644
--- a/api/server/controllers/agents/resume.js
+++ b/api/server/controllers/agents/resume.js
@@ -1,3 +1,4 @@
+const { randomUUID } = require('crypto');
const { logger } = require('@librechat/data-schemas');
const { Constants, EModelEndpoint } = require('librechat-data-provider');
const {
@@ -729,6 +730,7 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// the user when they retry the still-paused approval. Release the slot on that path too.
let claimed;
let checkpointGeneration;
+ const providerExecutionId = randomUUID();
try {
checkpointGeneration = await checkpointGenerationPromise;
/** The CAS that reopens steering must also publish THIS owner's seal
@@ -739,6 +741,8 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
pendingAction.actionId,
{
preemptCapable: isSteerPreemptSupported(),
+ providerExecutionId,
+ providerDrained: true,
...(resolvedAskUserQuestion && { resolvedAskUserQuestions }),
},
job.createdAt,
@@ -902,6 +906,17 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
let pausePersistenceFailed = false;
let pausePersistenceFailureFinalized = false;
try {
+ if (
+ !(await GenerationJobManager.beginProviderExecution(
+ streamId,
+ job.createdAt,
+ providerExecutionId,
+ ))
+ ) {
+ throw Object.assign(new Error('Generation stopped before provider resume'), {
+ code: 'RUN_REPLACED',
+ });
+ }
const result = await initializeClient({
req,
res,
@@ -1091,17 +1106,27 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
}
}
} finally {
- // Tear down the MCP request-context store seeded before the ACK (parity with
- // request.js's finishResumableRequest). No-op if it was never seeded.
- await cleanupMCPRequestContextForReq(req);
- // Release the concurrency slot taken above — UNLESS handleRunInterrupt already
- // released it on a re-pause (so a fast /resume isn't 429'd). On a normal finish or
- // error it didn't, so release here. A re-pause re-acquires its own slot next resume.
- if (!client?.pendingRequestReleased) {
- await decrementPendingRequest(userId);
- }
- if (client) {
- disposeClient(client);
+ try {
+ // Tear down the MCP request-context store seeded before the ACK (parity with
+ // request.js's finishResumableRequest). No-op if it was never seeded.
+ await cleanupMCPRequestContextForReq(req);
+ // Release the concurrency slot taken above — UNLESS handleRunInterrupt already
+ // released it on a re-pause (so a fast /resume isn't 429'd). On a normal finish or
+ // error it didn't, so release here. A re-pause re-acquires its own slot next resume.
+ if (!client?.pendingRequestReleased) {
+ await decrementPendingRequest(userId);
+ }
+ if (client) {
+ disposeClient(client);
+ }
+ } finally {
+ await GenerationJobManager.markProviderExecutionDrained?.(
+ streamId,
+ job.createdAt,
+ providerExecutionId,
+ ).catch((drainError) => {
+ logger.warn('[ResumeAgentController] Failed to record provider drain', drainError);
+ });
}
}
};
diff --git a/api/server/experimental.js b/api/server/experimental.js
index a259a8e31c..387e6940a5 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -25,6 +25,7 @@ const {
preAuthTenantMiddleware,
requestContextMiddleware,
configureServerTimeouts,
+ setupGracefulShutdown,
configureMessageFilterRegexValidator,
configureFileConfigRegexEngine,
} = require('@librechat/api');
@@ -34,6 +35,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilities
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { startExpiredFileSweep } = require('./services/Files/process');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
+const { initializeAgentTriggerService } = require('./services/Agents/triggers');
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
const { updateInterfacePermissions: updateInterfacePerms } = require('@librechat/api');
const {
@@ -159,6 +161,8 @@ if (cluster.isMaster) {
const listeningWorkers = new Set();
let retentionSweepWorkerId = null;
const startTime = Date.now();
+ let shuttingDown = false;
+ let remainingShutdownWorkers = 0;
const assignRetentionSweepWorker = () => {
if (retentionSweepWorkerId && cluster.workers[retentionSweepWorkerId]) {
@@ -229,15 +233,32 @@ if (cluster.isMaster) {
logger.error(
`Worker ${worker.process.pid} died (${activeWorkers}/${workers}). Code: ${code}, Signal: ${signal}`,
);
+ if (shuttingDown) {
+ remainingShutdownWorkers = Math.max(0, remainingShutdownWorkers - 1);
+ if (remainingShutdownWorkers === 0) {
+ process.exit(0);
+ }
+ return;
+ }
logger.info('Starting a new worker to replace it...');
cluster.fork();
});
/** Graceful shutdown on SIGTERM/SIGINT */
const shutdown = () => {
+ if (shuttingDown) {
+ return;
+ }
+ shuttingDown = true;
logger.info('Master received shutdown signal, terminating workers...');
- for (const id in cluster.workers) {
- cluster.workers[id].kill();
+ const liveWorkers = Object.values(cluster.workers).filter(Boolean);
+ remainingShutdownWorkers = liveWorkers.length;
+ if (remainingShutdownWorkers === 0) {
+ process.exit(0);
+ return;
+ }
+ for (const worker of liveWorkers) {
+ worker.kill();
}
setTimeout(() => {
logger.info('Forcing shutdown after timeout');
@@ -499,6 +520,7 @@ if (cluster.isMaster) {
await initializeMCPs();
await initializeOAuthReconnectManager();
await checkMigrations();
+ await initializeAgentTriggerService({ address: server.address() });
} catch (initErr) {
logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);
process.exit(1);
@@ -512,6 +534,7 @@ if (cluster.isMaster) {
headersTimeout: server.headersTimeout,
requestTimeout: server.requestTimeout,
});
+ setupGracefulShutdown(server);
};
startServer().catch((err) => {
diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.js
index 2287011154..c4364fb6b8 100644
--- a/api/server/experimental.spec.js
+++ b/api/server/experimental.spec.js
@@ -13,6 +13,16 @@ describe('Experimental server configuration', () => {
expect(listenIndex).toBeLessThan(timeoutConfigIndex);
});
+ it('lets each worker drain registered services before cluster shutdown', () => {
+ const listenIndex = source.indexOf('const server = app.listen');
+ const gracefulShutdownIndex = source.indexOf('setupGracefulShutdown(server);');
+
+ expect(gracefulShutdownIndex).toBeGreaterThan(-1);
+ expect(listenIndex).toBeLessThan(gracefulShutdownIndex);
+ expect(source).toContain('if (shuttingDown) {');
+ expect(source).toMatch(/if \(shuttingDown\) \{[\s\S]*?return;[\s\S]*?Starting a new worker/);
+ });
+
it('runs cross-tenant startup work in the system context', () => {
expect(source).toContain('await runAsSystem(seedDatabase);');
expect(source).toMatch(
diff --git a/api/server/index.js b/api/server/index.js
index 6732f1eaf3..87aed78629 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -56,6 +56,7 @@ const initializeOAuthReconnectManager = require('./services/initializeOAuthRecon
const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
+const { initializeAgentTriggerService } = require('./services/Agents/triggers');
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
const { startExpiredFileSweep } = require('./services/Files/process');
const { checkMigrations } = require('./services/start/migration');
@@ -397,6 +398,7 @@ const startServer = async () => {
if (inspectFlags || isEnabled(process.env.MEM_DIAG)) {
memoryDiagnostics.start();
}
+ await initializeAgentTriggerService({ address: server.address() });
serverReady = true;
logger.info('Server readiness checks passing.');
} catch (initErr) {
diff --git a/api/server/index.metrics.spec.js b/api/server/index.metrics.spec.js
index b611488cd9..6b95104c3d 100644
--- a/api/server/index.metrics.spec.js
+++ b/api/server/index.metrics.spec.js
@@ -34,6 +34,10 @@ jest.mock('~/config', () => ({
}),
}));
+jest.mock('~/server/services/Agents/triggers', () => ({
+ initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),
+}));
+
describe('Server metrics route', () => {
jest.setTimeout(30_000);
diff --git a/api/server/index.spec.js b/api/server/index.spec.js
index bdc249bcc7..6461a148e3 100644
--- a/api/server/index.spec.js
+++ b/api/server/index.spec.js
@@ -35,6 +35,10 @@ jest.mock('~/config', () => ({
}),
}));
+jest.mock('~/server/services/Agents/triggers', () => ({
+ initializeAgentTriggerService: jest.fn().mockResolvedValue(undefined),
+}));
+
jest.mock(
'@librechat/api/telemetry',
() => ({
@@ -164,6 +168,14 @@ describe('Startup readiness wiring', () => {
expect(agentsRouteIndex).toBeGreaterThan(-1);
expect(readinessGateIndex).toBeLessThan(agentsRouteIndex);
});
+
+ it('awaits durable trigger delivery before reporting readiness', () => {
+ const triggerDeliveryIndex = source.indexOf('await initializeAgentTriggerService(');
+ const readyIndex = source.indexOf('serverReady = true;');
+
+ expect(triggerDeliveryIndex).toBeGreaterThan(-1);
+ expect(readyIndex).toBeGreaterThan(triggerDeliveryIndex);
+ });
});
describe('Server Configuration', () => {
diff --git a/api/server/middleware/__tests__/requireJwtAuth.spec.js b/api/server/middleware/__tests__/requireJwtAuth.spec.js
index 08f779fac0..bbadd5fb04 100644
--- a/api/server/middleware/__tests__/requireJwtAuth.spec.js
+++ b/api/server/middleware/__tests__/requireJwtAuth.spec.js
@@ -253,6 +253,30 @@ describe('requireJwtAuth tenant context chaining', () => {
expect(logger.warn).not.toHaveBeenCalled();
});
+ it('preserves the typed account-deletion fence in an authentication rejection', () => {
+ const req = mockReq(undefined, {
+ _mockStrategies: {
+ jwt: {
+ user: false,
+ info: {
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ },
+ status: 401,
+ },
+ },
+ });
+ const res = mockRes();
+
+ requireJwtAuth(req, res, jest.fn());
+
+ expect(res.status).toHaveBeenCalledWith(401);
+ expect(res.json).toHaveBeenCalledWith({
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ });
+ });
+
it('logs OpenID JWT expiry when JWT fallback succeeds', () => {
isEnabled.mockReturnValue(true);
mockRegisteredStrategies.add('openidJwt');
diff --git a/api/server/middleware/requireJwtAuth.js b/api/server/middleware/requireJwtAuth.js
index 251fef9958..74232012f2 100644
--- a/api/server/middleware/requireJwtAuth.js
+++ b/api/server/middleware/requireJwtAuth.js
@@ -17,6 +17,7 @@ const hasPassportStrategy = (strategy) =>
const getAuthenticatedUserId = (user) => user?.id?.toString?.() ?? user?._id?.toString?.();
const refreshCloudFrontCookies =
maybeRefreshCloudFrontAuthCookiesMiddleware ?? ((_req, _res, next) => next());
+const ACCOUNT_DELETION_CODE = 'ACCOUNT_DELETION_IN_PROGRESS';
const getAuthTokenSource = (req) => {
const authorization = req.headers.authorization;
@@ -166,6 +167,7 @@ const requireJwtAuth = (req, res, next) => {
logAuthenticationFailure({ strategy, info, status, err });
return res.status(status || 401).json({
message: info?.message || 'Unauthorized',
+ ...(info?.code === ACCOUNT_DELETION_CODE && { code: ACCOUNT_DELETION_CODE }),
});
}
if (strategy === 'openidJwt' && getAuthenticatedUserId(user) !== openIdReuseUserId) {
diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js
index 20d4eb1797..0aa157c179 100644
--- a/api/server/routes/admin/users.js
+++ b/api/server/routes/admin/users.js
@@ -3,6 +3,12 @@ const { createAdminUsersHandlers } = require('@librechat/api');
const { SystemCapabilities } = require('@librechat/data-schemas');
const { requireCapability } = require('~/server/middleware/roles/capabilities');
const { requireJwtAuth } = require('~/server/middleware');
+const {
+ drainAgentTriggerDeliveriesForUser,
+ prepareAgentTriggerUserPurge,
+ cancelAgentTriggerUserPurge,
+ purgeAgentTriggerDeliveriesForUser,
+} = require('~/server/services/Agents/triggers');
const db = require('~/models');
const router = express.Router();
@@ -14,6 +20,12 @@ const requireReadUsers = requireCapability(SystemCapabilities.READ_USERS);
const handlers = createAdminUsersHandlers({
findUsers: db.findUsers,
countUsers: db.countUsers,
+ beginAgentTriggerUserDeletion: db.beginAgentTriggerUserDeletion,
+ cancelAgentTriggerUserDeletion: db.cancelAgentTriggerUserDeletion,
+ drainAgentTriggerDeliveriesForUser,
+ prepareAgentTriggerUserPurge,
+ cancelAgentTriggerUserPurge,
+ purgeAgentTriggerDeliveriesForUser,
deleteUserById: db.deleteUserById,
deleteConfig: db.deleteConfig,
deleteAclEntries: db.deleteAclEntries,
diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.js
index 987fba06df..ee3ed4e7bb 100644
--- a/api/server/routes/agents/index.js
+++ b/api/server/routes/agents/index.js
@@ -13,6 +13,8 @@ const {
attachAskUserQuestionAnswers,
attachAskUserQuestionArgs,
createMessageFilterPii,
+ isAgentTriggerRequest,
+ exemptAgentTriggerFromIpLimiter,
} = require('@librechat/api');
const { createSseStreamTelemetry } = require('@librechat/api/telemetry');
const { logger } = require('@librechat/data-schemas');
@@ -41,6 +43,10 @@ const chat = require('./chat');
const { LIMIT_MESSAGE_IP, LIMIT_MESSAGE_USER } = process.env ?? {};
+/** Applies `limiter` unless this trusted loopback request should skip it. */
+const unless = (isExempt, limiter) => (req, res, next) =>
+ isExempt(req) ? next() : limiter(req, res, next);
+
/** Untenanted jobs (pre-multi-tenancy) remain accessible if the userId check passes. */
function hasTenantMismatch(job, user) {
return job.metadata?.tenantId != null && job.metadata.tenantId !== user.tenantId;
@@ -109,6 +115,12 @@ router.use('/v1/responses', responses);
router.use('/v1', openai);
router.use(requireJwtAuth);
+// Capture the short-lived trigger identity immediately after authentication. Downstream
+// middleware reads this stable decision instead of re-verifying an expired token.
+router.use((req, _res, next) => {
+ req._isAgentTrigger = isAgentTriggerRequest(req);
+ next();
+});
router.use(checkBan);
router.use(uaParser);
@@ -899,7 +911,7 @@ router.post('/chat/abort', configMiddleware, async (req, res, next) => {
*/
const steerLimiters = [];
if (isEnabled(LIMIT_MESSAGE_IP)) {
- steerLimiters.push(messageIpLimiter);
+ steerLimiters.push(unless(exemptAgentTriggerFromIpLimiter, messageIpLimiter));
}
if (isEnabled(LIMIT_MESSAGE_USER)) {
steerLimiters.push(messageUserLimiter);
@@ -961,7 +973,7 @@ const chatRouter = express.Router();
chatRouter.use(configMiddleware);
if (isEnabled(LIMIT_MESSAGE_IP)) {
- chatRouter.use(messageIpLimiter);
+ chatRouter.use(unless(exemptAgentTriggerFromIpLimiter, messageIpLimiter));
}
if (isEnabled(LIMIT_MESSAGE_USER)) {
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
new file mode 100644
index 0000000000..7beb92f8ee
--- /dev/null
+++ b/api/server/services/Agents/triggers.js
@@ -0,0 +1,21 @@
+const { createAgentTriggerService } = require('@librechat/api');
+const methods = require('~/models');
+
+const service = createAgentTriggerService({
+ methods,
+ isPrincipalActive: methods.isAgentTriggerPrincipalActive,
+});
+
+module.exports = {
+ initializeAgentTriggerService: service.initialize,
+ stopAgentTriggerService: service.stop,
+ dispatchAgentTrigger: service.dispatch,
+ enqueueAgentTrigger: service.enqueue,
+ getAgentTriggerDelivery: service.getDelivery,
+ getAgentTriggerDeadLetters: service.getDeadLetters,
+ requeueAgentTrigger: service.requeue,
+ drainAgentTriggerDeliveriesForUser: service.drainUser,
+ prepareAgentTriggerUserPurge: service.prepareUserPurge,
+ cancelAgentTriggerUserPurge: service.cancelUserPurge,
+ purgeAgentTriggerDeliveriesForUser: service.purgeUser,
+};
diff --git a/api/strategies/jwtStrategy.js b/api/strategies/jwtStrategy.js
index 7a41f553e2..d5de52276b 100644
--- a/api/strategies/jwtStrategy.js
+++ b/api/strategies/jwtStrategy.js
@@ -1,25 +1,59 @@
-const { logger } = require('@librechat/data-schemas');
+const { AGENT_TRIGGER_SCOPE } = require('@librechat/api');
+const { logger, runAsSystem } = require('@librechat/data-schemas');
const { SystemRoles } = require('librechat-data-provider');
const { Strategy: JwtStrategy, ExtractJwt } = require('passport-jwt');
const { getUserById, updateUser } = require('~/models');
+const AGENT_TRIGGER_ADMISSION_PATHS = ['/api/agents/chat/agents', '/api/agents/chat/steer/deliver'];
+
+/** A trigger-scoped bearer is deliberately useless outside the two internal
+ * admission routes that the execution host calls. */
+function isAgentTriggerAdmissionRequest(req) {
+ if (req?.method !== 'POST' || req?.headers?.['x-lc-agent-trigger'] !== '1') {
+ return false;
+ }
+ try {
+ const pathname = new URL(req.originalUrl ?? req.url ?? '', 'http://localhost').pathname;
+ return AGENT_TRIGGER_ADMISSION_PATHS.some((path) => pathname.endsWith(path));
+ } catch {
+ return false;
+ }
+}
+
// JWT strategy
const jwtLogin = () =>
new JwtStrategy(
{
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
secretOrKey: process.env.JWT_SECRET,
+ passReqToCallback: true,
},
- async (payload, done) => {
+ async (req, payload, done) => {
try {
- const user = await getUserById(payload?.id, '-password -__v -totpSecret -backupCodes');
+ if (payload?.scope === AGENT_TRIGGER_SCOPE && !isAgentTriggerAdmissionRequest(req)) {
+ done(null, false, { message: 'Agent trigger token is not valid for this endpoint' });
+ return;
+ }
+ const user = await runAsSystem(() =>
+ getUserById(
+ payload?.id,
+ '-password -__v -totpSecret -backupCodes +agentTriggerDeletionStartedAt',
+ ),
+ );
+ if (user?.agentTriggerDeletionStartedAt != null) {
+ done(null, false, {
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ });
+ return;
+ }
if (user) {
user.id = user._id.toString();
/** Absent on the full doc means local user; null skips getUserPrincipals' fallback lookup */
user.idOnTheSource ??= null;
if (!user.role) {
user.role = SystemRoles.USER;
- await updateUser(user.id, { role: user.role });
+ await runAsSystem(() => updateUser(user.id, { role: user.role }));
}
done(null, user);
} else {
diff --git a/api/strategies/jwtStrategy.spec.js b/api/strategies/jwtStrategy.spec.js
index 3a8b83812d..da405ecb0c 100644
--- a/api/strategies/jwtStrategy.spec.js
+++ b/api/strategies/jwtStrategy.spec.js
@@ -1,8 +1,11 @@
const { SystemRoles } = require('librechat-data-provider');
let capturedVerifyCallback;
+let capturedStrategyOptions;
+const mockRunAsSystem = jest.fn((callback) => callback());
jest.mock('passport-jwt', () => ({
Strategy: jest.fn((opts, verifyCallback) => {
+ capturedStrategyOptions = opts;
capturedVerifyCallback = verifyCallback;
return { name: 'jwt' };
}),
@@ -13,6 +16,11 @@ jest.mock('passport-jwt', () => ({
jest.mock('@librechat/data-schemas', () => ({
logger: { info: jest.fn(), warn: jest.fn(), debug: jest.fn(), error: jest.fn() },
+ runAsSystem: mockRunAsSystem,
+}));
+
+jest.mock('@librechat/api', () => ({
+ AGENT_TRIGGER_SCOPE: 'agent_trigger',
}));
jest.mock('~/models', () => ({
@@ -23,9 +31,18 @@ jest.mock('~/models', () => ({
const jwtLogin = require('./jwtStrategy');
const { getUserById, updateUser } = require('~/models');
-function invokeVerify(payload) {
+function request(overrides = {}) {
+ return {
+ method: 'GET',
+ originalUrl: '/api/user',
+ headers: {},
+ ...overrides,
+ };
+}
+
+function invokeVerify(payload, req = request()) {
return new Promise((resolve, reject) => {
- capturedVerifyCallback(payload, (err, user, info) => {
+ capturedVerifyCallback(req, payload, (err, user, info) => {
if (err) {
return reject(err);
}
@@ -51,6 +68,23 @@ describe('jwtStrategy', () => {
expect(user.id).toBe('user-1');
expect(user.idOnTheSource).toBeNull();
+ expect(capturedStrategyOptions.passReqToCallback).toBe(true);
+ });
+
+ it('runs globally scoped authentication reads and writes in system context', async () => {
+ getUserById.mockResolvedValue({
+ _id: { toString: () => 'user-system' },
+ });
+
+ const { user } = await invokeVerify({ id: 'user-system' });
+
+ expect(user).toMatchObject({ id: 'user-system', role: SystemRoles.USER });
+ expect(mockRunAsSystem).toHaveBeenCalledTimes(2);
+ expect(getUserById).toHaveBeenCalledWith(
+ 'user-system',
+ '-password -__v -totpSecret -backupCodes +agentTriggerDeletionStartedAt',
+ );
+ expect(updateUser).toHaveBeenCalledWith('user-system', { role: SystemRoles.USER });
});
it('preserves a stored idOnTheSource for federated users', async () => {
@@ -72,4 +106,81 @@ describe('jwtStrategy', () => {
expect(user).toBe(false);
});
+
+ it('accepts a scoped trigger token only on an exact marked admission route', async () => {
+ getUserById.mockResolvedValue({
+ _id: { toString: () => 'user-3' },
+ role: SystemRoles.USER,
+ });
+
+ const { user } = await invokeVerify(
+ { id: 'user-3', scope: 'agent_trigger' },
+ request({
+ method: 'POST',
+ originalUrl: '/trusted/base/api/agents/chat/agents?source=delivery',
+ headers: { 'x-lc-agent-trigger': '1' },
+ }),
+ );
+
+ expect(user.id).toBe('user-3');
+ });
+
+ it.each([
+ ['an unrelated route', request({ method: 'POST', originalUrl: '/api/user' })],
+ [
+ 'a missing transport marker',
+ request({ method: 'POST', originalUrl: '/api/agents/chat/agents' }),
+ ],
+ [
+ 'a non-POST request',
+ request({
+ method: 'GET',
+ originalUrl: '/api/agents/chat/steer/deliver',
+ headers: { 'x-lc-agent-trigger': '1' },
+ }),
+ ],
+ ])('rejects a scoped trigger token on %s', async (_label, req) => {
+ const { user, info } = await invokeVerify({ id: 'user-3', scope: 'agent_trigger' }, req);
+
+ expect(user).toBe(false);
+ expect(info).toEqual({ message: 'Agent trigger token is not valid for this endpoint' });
+ expect(getUserById).not.toHaveBeenCalled();
+ });
+
+ it('rejects ordinary and trigger-scoped tokens while account deletion is fenced', async () => {
+ getUserById.mockResolvedValue({
+ _id: { toString: () => 'user-3' },
+ role: SystemRoles.USER,
+ agentTriggerDeletionStartedAt: new Date('2026-08-17T12:00:00.000Z'),
+ });
+
+ const ordinary = await invokeVerify({ id: 'user-3' });
+ const trigger = await invokeVerify(
+ { id: 'user-3', scope: 'agent_trigger' },
+ request({
+ method: 'POST',
+ originalUrl: '/api/agents/chat/steer/deliver',
+ headers: { 'x-lc-agent-trigger': '1' },
+ }),
+ );
+
+ expect(ordinary).toEqual({
+ user: false,
+ info: {
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ },
+ });
+ expect(trigger).toEqual({
+ user: false,
+ info: {
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ },
+ });
+ expect(getUserById).toHaveBeenCalledWith(
+ 'user-3',
+ '-password -__v -totpSecret -backupCodes +agentTriggerDeletionStartedAt',
+ );
+ });
});
diff --git a/api/strategies/openIdJwtStrategy.js b/api/strategies/openIdJwtStrategy.js
index 3606fe474b..5cd55020f1 100644
--- a/api/strategies/openIdJwtStrategy.js
+++ b/api/strategies/openIdJwtStrategy.js
@@ -1,6 +1,6 @@
const cookies = require('cookie');
const jwksRsa = require('jwks-rsa');
-const { logger, getTenantId } = require('@librechat/data-schemas');
+const { logger, getTenantId, runAsSystem } = require('@librechat/data-schemas');
const { CacheKeys, SystemRoles } = require('librechat-data-provider');
const { Strategy: JwtStrategy, ExtractJwt } = require('passport-jwt');
const {
@@ -18,7 +18,7 @@ const {
getHttpsProxyAgent,
math,
} = require('@librechat/api');
-const { updateUser, findUser } = require('~/models');
+const { updateUser, findUser, isAgentTriggerPrincipalActive } = require('~/models');
const getLogStores = require('~/cache/getLogStores');
const getOpenIdJwtAudience = () => {
@@ -173,6 +173,13 @@ const openIdJwtLogin = (openIdConfig) => {
if (user) {
user.id = user._id.toString();
+ if (!(await runAsSystem(() => isAgentTriggerPrincipalActive(user.id)))) {
+ done(null, false, {
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ });
+ return;
+ }
/** Absent on the full doc means local user; null skips getUserPrincipals' fallback lookup */
user.idOnTheSource ??= null;
diff --git a/api/strategies/openIdJwtStrategy.spec.js b/api/strategies/openIdJwtStrategy.spec.js
index a1993aa5c7..e893628edb 100644
--- a/api/strategies/openIdJwtStrategy.spec.js
+++ b/api/strategies/openIdJwtStrategy.spec.js
@@ -10,6 +10,7 @@ const mockAuthUserDocCacheStore = {
};
const mockGetLogStores = jest.fn(() => mockAuthUserDocCacheStore);
const mockGetTenantId = jest.fn();
+const mockRunAsSystem = jest.fn((callback) => callback());
jest.mock('passport-jwt', () => ({
Strategy: jest.fn((opts, verifyCallback) => {
capturedStrategyOptions = opts;
@@ -29,6 +30,7 @@ jest.mock('https-proxy-agent', () => ({
jest.mock('@librechat/data-schemas', () => ({
logger: { info: jest.fn(), warn: jest.fn(), debug: jest.fn(), error: jest.fn() },
getTenantId: mockGetTenantId,
+ runAsSystem: mockRunAsSystem,
}));
jest.mock('@librechat/api', () => ({
isEnabled: jest.fn(() => false),
@@ -48,6 +50,7 @@ jest.mock('@librechat/api', () => ({
jest.mock('~/models', () => ({
findUser: jest.fn(),
updateUser: jest.fn(),
+ isAgentTriggerPrincipalActive: jest.fn(() => true),
}));
jest.mock('~/server/services/Files/strategies', () => ({
getStrategyFunctions: jest.fn(() => ({
@@ -69,7 +72,7 @@ const {
setCachedAuthUserDoc,
} = require('@librechat/api');
const openIdJwtLogin = require('./openIdJwtStrategy');
-const { findUser, updateUser } = require('~/models');
+const { findUser, updateUser, isAgentTriggerPrincipalActive } = require('~/models');
function resetAuthUserDocCacheMocks() {
mockGetTenantId.mockReturnValue(undefined);
@@ -87,6 +90,8 @@ function resetAuthUserDocCacheMocks() {
beforeEach(() => {
resetAuthUserDocCacheMocks();
+ mockRunAsSystem.mockClear();
+ isAgentTriggerPrincipalActive.mockResolvedValue(true);
});
function withEnv(env, callback) {
@@ -451,6 +456,32 @@ describe('openIdJwtStrategy – auth user document cache', () => {
expect(invalidateCachedAuthUserDoc).not.toHaveBeenCalled();
});
+ it('rejects a cached OpenID user while account deletion is fenced', async () => {
+ mockGetTenantId.mockReturnValue('tenant-a');
+ getAuthUserDocCacheMode.mockReturnValue('on');
+ getCachedAuthUserDoc.mockResolvedValue({
+ _id: 'cached-user',
+ role: SystemRoles.USER,
+ provider: 'openid',
+ tenantId: 'tenant-a',
+ });
+ isAgentTriggerPrincipalActive.mockResolvedValue(false);
+
+ const result = await invokeVerify(req, payload);
+
+ expect(result).toEqual({
+ user: false,
+ info: {
+ message: 'Account deletion is in progress',
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ },
+ });
+ expect(findOpenIDUser).not.toHaveBeenCalled();
+ expect(mockRunAsSystem).toHaveBeenCalledWith(expect.any(Function));
+ expect(isAgentTriggerPrincipalActive).toHaveBeenCalledWith('cached-user');
+ expect(setCachedAuthUserDoc).not.toHaveBeenCalled();
+ });
+
it('populates the cache after a miss with the fresh user document', async () => {
getAuthUserDocCacheMode.mockReturnValue('on');
getCachedAuthUserDoc.mockResolvedValue(undefined);
diff --git a/config/delete-user.js b/config/delete-user.js
index 66e325d1ee..fec8df7590 100644
--- a/config/delete-user.js
+++ b/config/delete-user.js
@@ -3,6 +3,7 @@
// @ts-nocheck
const path = require('path');
const mongoose = require('mongoose');
+const { createModels, createMethods, runAsSystem } = require('@librechat/data-schemas');
const {
Key,
User,
@@ -27,11 +28,23 @@ const {
Transaction,
Conversation,
ConversationTag,
-} = require('@librechat/data-schemas').createModels(mongoose);
+} = createModels(mongoose);
require('module-alias')({ base: path.resolve(__dirname, '..', 'api') });
+const {
+ GenerationJobManager,
+ createStreamServices,
+ waitForKeyvRedisClient,
+} = require('@librechat/api');
+const getLogStores = require('~/cache/getLogStores');
const { askQuestion, silentExit } = require('./helpers');
const connect = require('./connect');
+const TRIGGER_DRAIN_TIMEOUT_MS = 35_000;
+const TRIGGER_DRAIN_POLL_MS = 100;
+const methods = createMethods(mongoose, { getCache: getLogStores });
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
async function gracefulExit(code = 0) {
try {
await mongoose.disconnect();
@@ -76,41 +89,157 @@ async function gracefulExit(code = 0) {
const uid = user._id.toString();
- // 5) Build and run deletion tasks
- const tasks = [
- Action.deleteMany({ user: uid }),
- Agent.deleteMany({ author: uid }),
- AgentApiKey.deleteMany({ user: uid }),
- Assistant.deleteMany({ user: uid }),
- Balance.deleteMany({ user: uid }),
- ConversationTag.deleteMany({ user: uid }),
- Conversation.deleteMany({ user: uid }),
- Message.deleteMany({ user: uid }),
- File.deleteMany({ user: uid }),
- Key.deleteMany({ userId: uid }),
- MemoryEntry.deleteMany({ userId: uid }),
- PluginAuth.deleteMany({ userId: uid }),
- Prompt.deleteMany({ author: uid }),
- PromptGroup.deleteMany({ author: uid }),
- Preset.deleteMany({ user: uid }),
- Session.deleteMany({ user: uid }),
- SharedLink.deleteMany({ user: uid }),
- ToolCall.deleteMany({ user: uid }),
- Token.deleteMany({ userId: uid }),
- AclEntry.deleteMany({ principalId: user._id }),
- ];
-
- if (deleteTx) {
- tasks.push(Transaction.deleteMany({ user: uid }));
+ // The CLI can coordinate live generation aborts only through the shared
+ // Redis stream store. Without it, require an explicit offline assertion.
+ let streamServices;
+ try {
+ await waitForKeyvRedisClient();
+ streamServices = createStreamServices();
+ } catch (error) {
+ console.yellow(
+ `Shared Redis generation coordination is unreachable: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ streamServices = createStreamServices({ useRedis: false });
}
+ const hasSharedGenerationStore = streamServices.isRedis;
+ let allProcessesStopped = false;
+ if (!hasSharedGenerationStore) {
+ const confirmOffline = await askQuestion(
+ 'Shared Redis generation coordination is unavailable. Confirm ALL LibreChat app, worker, and other deletion CLI processes are stopped before continuing. (y/N)',
+ );
+ if (confirmOffline.toLowerCase() !== 'y') {
+ console.yellow('Aborted. Stop every LibreChat process or enable Redis stream coordination.');
+ return gracefulExit(1);
+ }
+ allProcessesStopped = true;
+ } else {
+ GenerationJobManager.configure({ ...streamServices, cleanupOnComplete: false });
+ GenerationJobManager.initialize();
+ }
+
+ let deletionFence;
+ let userDeleted = false;
+
+ try {
+ deletionFence = new Date();
+ let fenceState = await runAsSystem(() =>
+ methods.beginAgentTriggerUserDeletion(uid, deletionFence),
+ );
+ if (fenceState === 'in_progress') {
+ deletionFence = undefined;
+ if (!allProcessesStopped) {
+ const confirmRecovery = await askQuestion(
+ 'An account-deletion fence already exists. Confirm ALL LibreChat app, worker, and other deletion CLI processes are stopped to recover it only if stale. (y/N)',
+ );
+ if (confirmRecovery.toLowerCase() !== 'y') {
+ throw new Error('Account deletion is already in progress');
+ }
+ allProcessesStopped = true;
+ }
+
+ const recoveredAt = new Date();
+ fenceState = await runAsSystem(() =>
+ methods.recoverStaleAgentTriggerUserDeletion(uid, recoveredAt),
+ );
+ if (fenceState !== 'acquired') {
+ throw new Error(
+ fenceState === 'missing'
+ ? 'User disappeared before stale-fence recovery'
+ : 'Account deletion is active or its fence is not stale enough to recover',
+ );
+ }
+ deletionFence = recoveredAt;
+ }
+ if (fenceState === 'missing') {
+ deletionFence = undefined;
+ }
+
+ if (deletionFence != null) {
+ await runAsSystem(() =>
+ methods.prepareAgentTriggerUserPurge(uid, deletionFence, user.tenantId),
+ );
+ if (hasSharedGenerationStore) {
+ const deadline = Date.now() + TRIGGER_DRAIN_TIMEOUT_MS;
+ while (
+ (await runAsSystem(() =>
+ methods.countActiveAgentTriggerDeliveriesByUser(uid, new Date()),
+ )) > 0
+ ) {
+ if (Date.now() >= deadline) {
+ throw new Error('Timed out draining active agent trigger deliveries');
+ }
+ await delay(TRIGGER_DRAIN_POLL_MS);
+ }
+ }
+ }
+
+ if (hasSharedGenerationStore) {
+ const cleanupBlockingAgentRuns = await GenerationJobManager.getCleanupBlockingJobIdsForUser(
+ uid,
+ user.tenantId,
+ );
+ await Promise.all(
+ cleanupBlockingAgentRuns.map((streamId) =>
+ GenerationJobManager.abortJob(streamId, { awaitProviderDrain: true }),
+ ),
+ );
+ }
- await Promise.all(tasks);
+ // 5) Build and run deletion tasks
+ const tasks = [
+ Action.deleteMany({ user: uid }),
+ Agent.deleteMany({ author: uid }),
+ AgentApiKey.deleteMany({ user: uid }),
+ Assistant.deleteMany({ user: uid }),
+ Balance.deleteMany({ user: uid }),
+ ConversationTag.deleteMany({ user: uid }),
+ Conversation.deleteMany({ user: uid }),
+ Message.deleteMany({ user: uid }),
+ File.deleteMany({ user: uid }),
+ Key.deleteMany({ userId: uid }),
+ MemoryEntry.deleteMany({ userId: uid }),
+ PluginAuth.deleteMany({ userId: uid }),
+ Prompt.deleteMany({ author: uid }),
+ PromptGroup.deleteMany({ author: uid }),
+ Preset.deleteMany({ user: uid }),
+ Session.deleteMany({ user: uid }),
+ SharedLink.deleteMany({ user: uid }),
+ ToolCall.deleteMany({ user: uid }),
+ Token.deleteMany({ userId: uid }),
+ AclEntry.deleteMany({ principalId: user._id }),
+ ];
- // 6) Remove user from all groups
- await Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });
+ if (deleteTx) {
+ tasks.push(Transaction.deleteMany({ user: uid }));
+ }
- // 7) Finally delete the user document itself
- await User.deleteOne({ _id: uid });
+ await Promise.all(tasks);
+
+ // 6) Remove user from all groups
+ await Group.updateMany({ memberIds: uid }, { $pullAll: { memberIds: [uid] } });
+
+ // 7) Finally delete the user document itself
+ const deletedUser = await runAsSystem(() => methods.deleteUserById(uid));
+ if (deletedUser.deletedCount !== 1) {
+ throw new Error('User disappeared before account deletion could commit');
+ }
+ userDeleted = true;
+ await runAsSystem(() => methods.deleteAgentTriggerDeliveriesByUser(uid));
+ } finally {
+ if (deletionFence != null && !userDeleted) {
+ await runAsSystem(() => methods.cancelAgentTriggerUserPurge(uid, deletionFence)).catch(
+ (error) => console.error('Failed to disarm trigger purge recovery:', error),
+ );
+ await runAsSystem(() => methods.cancelAgentTriggerUserDeletion(uid, deletionFence)).catch(
+ (error) => console.error('Failed to release account-deletion fence:', error),
+ );
+ }
+ if (hasSharedGenerationStore) {
+ await GenerationJobManager.destroy().catch((error) =>
+ console.error('Failed to close generation coordination:', error),
+ );
+ }
+ }
console.green(`✔ Successfully deleted user ${email} and all associated data.`);
if (!deleteTx) {
diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts
index 1d0e5fbac8..e7cae56ad4 100644
--- a/packages/api/src/admin/users.spec.ts
+++ b/packages/api/src/admin/users.spec.ts
@@ -53,6 +53,12 @@ function createDeps(overrides: Partial<AdminUsersDeps> = {}): AdminUsersDeps {
return {
findUsers: jest.fn().mockResolvedValue([]),
countUsers: jest.fn().mockResolvedValue(0),
+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('acquired'),
+ cancelAgentTriggerUserDeletion: jest.fn().mockResolvedValue(true),
+ drainAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined),
+ prepareAgentTriggerUserPurge: jest.fn().mockResolvedValue(undefined),
+ cancelAgentTriggerUserPurge: jest.fn().mockResolvedValue(true),
+ purgeAgentTriggerDeliveriesForUser: jest.fn().mockResolvedValue(undefined),
deleteUserById: jest
.fn()
.mockResolvedValue({ deletedCount: 1, message: 'User was deleted successfully.' }),
@@ -340,6 +346,19 @@ describe('createAdminUsersHandlers', () => {
expect(status).toHaveBeenCalledWith(200);
expect(json).toHaveBeenCalledWith({ message: 'User was deleted successfully.' });
+ expect(deps.beginAgentTriggerUserDeletion).toHaveBeenCalledWith(
+ validUserId,
+ expect.any(Date),
+ );
+ expect(deps.drainAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId);
+ expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
+ validUserId,
+ expect.any(Date),
+ undefined,
+ );
+ expect(deps.purgeAgentTriggerDeliveriesForUser).toHaveBeenCalledWith(validUserId);
+ expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
});
it('returns fallback message when result.message is empty', async () => {
@@ -464,6 +483,12 @@ describe('createAdminUsersHandlers', () => {
expect(status).toHaveBeenCalledWith(404);
expect(deps.deleteConfig).not.toHaveBeenCalled();
expect(deps.deleteAclEntries).not.toHaveBeenCalled();
+ expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
+ validUserId,
+ expect.any(Date),
+ );
+ expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));
});
it('returns 400 for invalid ObjectId', async () => {
@@ -500,6 +525,84 @@ describe('createAdminUsersHandlers', () => {
expect(status).toHaveBeenCalledWith(500);
expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
+ expect(deps.cancelAgentTriggerUserDeletion).toHaveBeenCalledWith(
+ validUserId,
+ expect.any(Date),
+ );
+ expect(deps.cancelAgentTriggerUserPurge).toHaveBeenCalledWith(validUserId, expect.any(Date));
+ expect(deps.purgeAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
+ });
+
+ it('does not delete while another deletion owns the trigger fence', async () => {
+ const deps = createDeps({
+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('in_progress'),
+ });
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ expect(status).toHaveBeenCalledWith(409);
+ expect(json).toHaveBeenCalledWith({ error: 'User deletion is already in progress' });
+ expect(deps.drainAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
+ expect(deps.deleteUserById).not.toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
+ });
+
+ it('returns not found without draining when the trigger fence principal is missing', async () => {
+ const deps = createDeps({
+ beginAgentTriggerUserDeletion: jest.fn().mockResolvedValue('missing'),
+ });
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ expect(status).toHaveBeenCalledWith(404);
+ expect(json).toHaveBeenCalledWith({ error: 'User not found' });
+ expect(deps.drainAgentTriggerDeliveriesForUser).not.toHaveBeenCalled();
+ expect(deps.deleteUserById).not.toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
+ });
+
+ it('drains before commit and purges only after the user is deleted', async () => {
+ const deps = createDeps();
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ const beginOrder = (deps.beginAgentTriggerUserDeletion as jest.Mock).mock
+ .invocationCallOrder[0];
+ const prepareOrder = (deps.prepareAgentTriggerUserPurge as jest.Mock).mock
+ .invocationCallOrder[0];
+ const drainOrder = (deps.drainAgentTriggerDeliveriesForUser as jest.Mock).mock
+ .invocationCallOrder[0];
+ const deleteOrder = (deps.deleteUserById as jest.Mock).mock.invocationCallOrder[0];
+ const purgeOrder = (deps.purgeAgentTriggerDeliveriesForUser as jest.Mock).mock
+ .invocationCallOrder[0];
+ expect(beginOrder).toBeLessThan(drainOrder);
+ expect(beginOrder).toBeLessThan(prepareOrder);
+ expect(prepareOrder).toBeLessThan(drainOrder);
+ expect(drainOrder).toBeLessThan(deleteOrder);
+ expect(deleteOrder).toBeLessThan(purgeOrder);
+ });
+
+ it('leaves durable purge recovery armed when immediate post-commit cleanup fails', async () => {
+ const deps = createDeps({
+ purgeAgentTriggerDeliveriesForUser: jest.fn().mockRejectedValue(new Error('db down')),
+ });
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status, json } = createReqRes({ params: { id: validUserId } });
+
+ await handlers.deleteUser(req, res);
+
+ expect(status).toHaveBeenCalledWith(500);
+ expect(json).toHaveBeenCalledWith({ error: 'Failed to delete user' });
+ expect(deps.deleteUserById).toHaveBeenCalledWith(validUserId);
+ expect(deps.prepareAgentTriggerUserPurge).toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserPurge).not.toHaveBeenCalled();
+ expect(deps.cancelAgentTriggerUserDeletion).not.toHaveBeenCalled();
});
});
});
diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts
index d24eb1b759..5a46d3c748 100644
--- a/packages/api/src/admin/users.ts
+++ b/packages/api/src/admin/users.ts
@@ -24,11 +24,25 @@ export interface AdminUsersDeps {
options?: { limit?: number; offset?: number; sort?: Record<string, 1 | -1> },
) => Promise<IUser[]>;
countUsers: (filter?: FilterQuery<IUser>) => Promise<number>;
+ beginAgentTriggerUserDeletion: (
+ userId: string,
+ startedAt: Date,
+ ) => Promise<'acquired' | 'in_progress' | 'missing'>;
+ cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;
+ drainAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;
+ prepareAgentTriggerUserPurge: (
+ userId: string,
+ fenceStartedAt: Date,
+ tenantId?: string,
+ ) => Promise<void>;
+ cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
+ purgeAgentTriggerDeliveriesForUser: (userId: string) => Promise<void>;
/**
* Thin data-layer delete — removes the User document only.
* Full cascade of user-owned resources (conversations, messages, files, tokens, etc.)
* is handled by `UserController.deleteUserController` in the self-delete flow.
- * This admin endpoint currently cascades Config and AclEntries.
+ * This admin endpoint fences durable triggers around the user commit and currently
+ * cascades Config and AclEntries.
* A future iteration should consolidate the full cascade into a shared service function.
*/
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
@@ -47,7 +61,19 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
searchUsers: (req: ServerRequest, res: Response) => Promise<Response>;
deleteUser: (req: ServerRequest, res: Response) => Promise<Response>;
} {
- const { findUsers, countUsers, deleteUserById, deleteConfig, deleteAclEntries } = deps;
+ const {
+ findUsers,
+ countUsers,
+ beginAgentTriggerUserDeletion,
+ cancelAgentTriggerUserDeletion,
+ drainAgentTriggerDeliveriesForUser,
+ prepareAgentTriggerUserPurge,
+ cancelAgentTriggerUserPurge,
+ purgeAgentTriggerDeliveriesForUser,
+ deleteUserById,
+ deleteConfig,
+ deleteAclEntries,
+ } = deps;
async function listUsersHandler(req: ServerRequest, res: Response) {
try {
@@ -126,8 +152,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
}
async function deleteUserHandler(req: ServerRequest, res: Response) {
+ let targetUserId: string | undefined;
+ let triggerDeletionFence: Date | undefined;
+ let userDeleted = false;
+
try {
const { id } = req.params as { id: string };
+ targetUserId = id;
if (!isValidObjectIdString(id)) {
return res.status(400).json({ error: 'Invalid user ID format' });
@@ -138,7 +169,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
return res.status(403).json({ error: 'Cannot delete your own account' });
}
- const [targetUser] = await findUsers({ _id: id }, 'role', { limit: 1 });
+ const [targetUser] = await findUsers({ _id: id }, 'role tenantId', { limit: 1 });
if (targetUser?.role === SystemRoles.ADMIN) {
const adminCount = await countUsers({ role: SystemRoles.ADMIN });
if (adminCount <= 1) {
@@ -146,11 +177,29 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
}
}
+ triggerDeletionFence = new Date();
+ const fenceState = await beginAgentTriggerUserDeletion(id, triggerDeletionFence);
+ if (fenceState === 'in_progress') {
+ triggerDeletionFence = undefined;
+ return res.status(409).json({ error: 'User deletion is already in progress' });
+ }
+ if (fenceState === 'missing') {
+ triggerDeletionFence = undefined;
+ return res.status(404).json({ error: 'User not found' });
+ }
+ await prepareAgentTriggerUserPurge(id, triggerDeletionFence, targetUser?.tenantId);
+ await drainAgentTriggerDeliveriesForUser(id);
+
const result = await deleteUserById(id);
if (result.deletedCount === 0) {
+ await cancelAgentTriggerUserPurge(id, triggerDeletionFence);
+ await cancelAgentTriggerUserDeletion(id, triggerDeletionFence);
+ triggerDeletionFence = undefined;
return res.status(404).json({ error: 'User not found' });
}
+ userDeleted = true;
+ await purgeAgentTriggerDeliveriesForUser(id);
if (targetUser?.role === SystemRoles.ADMIN) {
const remaining = await countUsers({ role: SystemRoles.ADMIN });
@@ -175,6 +224,18 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
return res.status(200).json({ message: result.message || 'User deleted successfully' });
} catch (error) {
+ if (targetUserId != null && triggerDeletionFence != null && !userDeleted) {
+ try {
+ await cancelAgentTriggerUserPurge(targetUserId, triggerDeletionFence);
+ } catch (purgeFenceError) {
+ logger.error('[adminUsers] failed to disarm trigger purge recovery:', purgeFenceError);
+ }
+ try {
+ await cancelAgentTriggerUserDeletion(targetUserId, triggerDeletionFence);
+ } catch (fenceError) {
+ logger.error('[adminUsers] failed to release trigger deletion fence:', fenceError);
+ }
+ }
logger.error('[adminUsers] deleteUser error:', error);
return res.status(500).json({ error: 'Failed to delete user' });
}
diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md
new file mode 100644
index 0000000000..1da00fe38e
--- /dev/null
+++ b/packages/api/src/agents/triggers/README.md
@@ -0,0 +1,67 @@
+# Agent trigger delivery
+
+This module is the trusted, source-neutral boundary for asynchronous agent work. A schedule,
+webhook, queue consumer, MCP integration, or internal event adapter produces the same versioned
+envelope and calls `enqueueAgentTrigger`; the adapter does not invoke an agent runtime directly.
+
+## Adapter contract
+
+- Authenticate and authorize the source before creating an envelope.
+- Strip credentials and transport secrets from `event.payload`.
+- Give each source event a stable `event.id`, and keep `deliveryId` stable for retries to one
+ target. A retry may use a fresh `requestId` and `receivedAt`.
+- Render bounded model input on the host. Infrastructure and routing remain server-controlled.
+- Use `orderingKey` only when deliveries must remain ordered across different event sources.
+ Without an override, ordering is scoped to the user, source, mode, agent, and conversation.
+
+```js
+const { createAgentTriggerEnvelope } = require('@librechat/api');
+const { enqueueAgentTrigger } = require('~/server/services/Agents/triggers');
+
+await enqueueAgentTrigger(
+ createAgentTriggerEnvelope({
+ mode: 'fire',
+ requestId,
+ deliveryId,
+ receivedAt: Date.now(),
+ principal: { id: userId, role, tenantId },
+ event: {
+ id: eventId,
+ type: 'resource.ready',
+ occurredAt,
+ source: { id: webhookId, type: 'webhook' },
+ payload: sanitizedPayload,
+ },
+ target: { agentId },
+ input,
+ }),
+ { orderingKey: resourceId },
+);
+```
+
+## Guarantees
+
+- Mongo owns queue state, leases, retry history, and dead letters across restarts and replicas.
+- A fresh token fences every claim, including reclaims by the same process.
+- A delivery is at-least-once. Fire and steer admission reuse the envelope's stable idempotency
+ identity, so ambiguous retries do not duplicate accepted work.
+- Retryable failures use bounded exponential backoff and honor `Retry-After`. Invalid envelopes,
+ permanent authorization failures, and exhausted retries become durable dead letters.
+- Matching ordering lanes serialize sequence allocation and queue publication behind a
+ Mongo-fenced publisher. A staging row is durable before taking that fence, and any replica can
+ finish an abandoned publication before allocating the next sequence, so a later delivery can
+ never overtake the invisible gap. Dead letters are terminal and do not block later work; an
+ explicit requeue admits the dead letter as a new lane tail so it cannot overlap newer in-flight
+ work. Inactive lane counters are reclaimed once no staging, queued, leased, or dead delivery
+ remains.
+- Successful records expire after 90 days. Dead letters remain until explicitly requeued or
+ removed. Account deletion first fences admission and drains active leases without destroying
+ queued work. A delivery deferred by that fence releases its lease and restores the attempt it
+ reserved, so a rolled-back deletion cannot exhaust its retry budget. Before deleting the user,
+ every deletion path durably arms an exact-fence purge marker; payloads are purged only after the
+ user deletion commits, and every replica retries any orphaned post-commit marker until cleanup
+ succeeds. An abandoned deletion fence can be recovered through `config/delete-user.js` only
+ after an operator confirms every competing app, worker, and deletion CLI process is stopped.
+
+`getAgentTriggerDeadLetters` and `requeueAgentTrigger` are intentionally trusted in-process
+operations. Exposing them through an admin API requires a separate authorization and audit layer.
diff --git a/packages/api/src/agents/triggers/delivery.integration.spec.ts b/packages/api/src/agents/triggers/delivery.integration.spec.ts
new file mode 100644
index 0000000000..6bdf157c95
--- /dev/null
+++ b/packages/api/src/agents/triggers/delivery.integration.spec.ts
@@ -0,0 +1,102 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import { createMethods, createModels } from '@librechat/data-schemas';
+import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service';
+import type { AgentTriggerFetch } from './host';
+import { __resetShutdownStateForTests } from '../../app/shutdown';
+import { createAgentTriggerEnvelope } from './envelope';
+import { createAgentTriggerService } from './service';
+
+const DB_SETUP_TIMEOUT_MS = 60_000;
+let mongoServer: MongoMemoryServer;
+let service: AgentTriggerService | undefined;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri(), { autoIndex: false });
+ createModels(mongoose);
+}, DB_SETUP_TIMEOUT_MS);
+
+afterEach(async () => {
+ await service?.stop();
+ service = undefined;
+ __resetShutdownStateForTests();
+ await mongoose.models.AgentTriggerDelivery.deleteMany({});
+});
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+}, DB_SETUP_TIMEOUT_MS);
+
+const envelope = () =>
+ createAgentTriggerEnvelope({
+ mode: 'fire',
+ requestId: 'request-1',
+ deliveryId: 'delivery-1',
+ receivedAt: 20,
+ principal: { id: new mongoose.Types.ObjectId().toString(), tenantId: 'tenant-1' },
+ target: { agentId: 'agent-1' },
+ event: {
+ id: 'event-1',
+ type: 'resource.ready',
+ occurredAt: 10,
+ source: { id: 'source-1', type: 'webhook' },
+ },
+ input: 'Handle the ready resource.',
+ });
+
+async function eventuallySucceeded(deliveryKey: string) {
+ const deadline = Date.now() + 5_000;
+ while (Date.now() < deadline) {
+ const delivery = await service!.getDelivery(deliveryKey);
+ if (delivery?.status === 'succeeded') {
+ return delivery;
+ }
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ throw new Error('Timed out waiting for the queued trigger to succeed');
+}
+
+describe('durable trigger delivery integration', () => {
+ it('moves a trusted envelope through Mongo, the lease worker, and host admission', async () => {
+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
+ async () =>
+ new Response(
+ JSON.stringify({
+ status: 'started',
+ streamId: 'conversation-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 25,
+ }),
+ { status: 200, headers: { 'content-type': 'application/json' } },
+ ),
+ );
+ service = createAgentTriggerService({
+ methods: createMethods(mongoose) as ReturnType<typeof createMethods> &
+ AgentTriggerDeliveryPersistence,
+ fetch: fetcher,
+ mintToken: () => 'trigger-token',
+ deliveryOptions: { concurrency: 2, tickMs: 5, retryBaseMs: 5 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+
+ const receipt = await service.enqueue(envelope(), { orderingKey: 'resource-1' });
+ const delivered = await eventuallySucceeded(receipt.deliveryKey);
+
+ expect(fetcher).toHaveBeenCalledTimes(1);
+ expect(delivered).toMatchObject({
+ id: receipt.id,
+ status: 'succeeded',
+ attempts: 1,
+ result: {
+ mode: 'fire',
+ status: 'started',
+ conversationId: 'conversation-1',
+ },
+ history: [{ attempt: 1, outcome: 'succeeded' }],
+ });
+ });
+});
diff --git a/packages/api/src/agents/triggers/delivery.spec.ts b/packages/api/src/agents/triggers/delivery.spec.ts
new file mode 100644
index 0000000000..022c362e7c
--- /dev/null
+++ b/packages/api/src/agents/triggers/delivery.spec.ts
@@ -0,0 +1,108 @@
+import {
+ AgentTriggerDeliveryError,
+ MAX_AGENT_TRIGGER_ENVELOPE_BYTES,
+ prepareAgentTriggerDelivery,
+} from './delivery';
+import { createAgentTriggerEnvelope } from './envelope';
+
+const envelope = (overrides: Record<string, unknown> = {}) =>
+ createAgentTriggerEnvelope({
+ mode: 'fire',
+ requestId: 'request-1',
+ deliveryId: 'delivery-1',
+ receivedAt: 20,
+ principal: { id: '507f1f77bcf86cd799439011', tenantId: 'tenant-1' },
+ target: { agentId: 'agent-1' },
+ event: {
+ id: 'event-1',
+ type: 'resource.ready',
+ occurredAt: 10,
+ source: { id: 'source-1', type: 'webhook' },
+ payload: { b: 2, a: 1 },
+ },
+ input: 'Handle the ready resource.',
+ ...overrides,
+ });
+
+describe('prepareAgentTriggerDelivery', () => {
+ it('prepares a bounded persistent record with a default ordering lane', () => {
+ const availableAt = new Date('2026-08-17T12:00:00.000Z');
+ const prepared = prepareAgentTriggerDelivery(envelope(), { availableAt });
+
+ expect(prepared).toMatchObject({
+ deliveryKey: expect.stringMatching(/^trigger_[a-f0-9]{64}$/),
+ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/),
+ orderingKey: expect.stringMatching(/^trigger_lane_[a-f0-9]{64}$/),
+ user: '507f1f77bcf86cd799439011',
+ tenantId: 'tenant-1',
+ availableAt,
+ });
+ expect(prepared.availableAt).not.toBe(availableAt);
+ });
+
+ it('treats fresh ingress metadata and payload key order as the same retry', () => {
+ const first = prepareAgentTriggerDelivery(envelope());
+ const retried = prepareAgentTriggerDelivery(
+ envelope({
+ requestId: 'request-2',
+ receivedAt: 30,
+ event: {
+ id: 'event-1',
+ type: 'resource.ready',
+ occurredAt: 10,
+ source: { id: 'source-1', type: 'webhook' },
+ payload: { a: 1, b: 2 },
+ },
+ }),
+ );
+
+ expect(retried.deliveryKey).toBe(first.deliveryKey);
+ expect(retried.fingerprint).toBe(first.fingerprint);
+ expect(retried.orderingKey).toBe(first.orderingKey);
+ });
+
+ it('detects execution-content changes under the same delivery identity', () => {
+ const first = prepareAgentTriggerDelivery(envelope());
+ const changed = prepareAgentTriggerDelivery(envelope({ input: 'Do something else.' }));
+
+ expect(changed.deliveryKey).toBe(first.deliveryKey);
+ expect(changed.fingerprint).not.toBe(first.fingerprint);
+ });
+
+ it('supports an explicit cross-source ordering lane scoped to the principal', () => {
+ const first = prepareAgentTriggerDelivery(envelope(), { orderingKey: 'game-42' });
+ const second = prepareAgentTriggerDelivery(
+ envelope({
+ deliveryId: 'delivery-2',
+ event: {
+ id: 'event-2',
+ type: 'clock.tick',
+ occurredAt: 11,
+ source: { id: 'source-2', type: 'scheduler' },
+ },
+ }),
+ { orderingKey: ' game-42 ' },
+ );
+ const anotherUser = prepareAgentTriggerDelivery(
+ envelope({ principal: { id: '507f1f77bcf86cd799439012', tenantId: 'tenant-1' } }),
+ { orderingKey: 'game-42' },
+ );
+
+ expect(second.orderingKey).toBe(first.orderingKey);
+ expect(anotherUser.orderingKey).not.toBe(first.orderingKey);
+ });
+
+ it('rejects invalid scheduling metadata and oversized envelopes', () => {
+ expect(() =>
+ prepareAgentTriggerDelivery(envelope(), { availableAt: new Date(Number.NaN) }),
+ ).toThrow(AgentTriggerDeliveryError);
+ expect(() => prepareAgentTriggerDelivery(envelope(), { orderingKey: ' ' })).toThrow(
+ AgentTriggerDeliveryError,
+ );
+ expect(() =>
+ prepareAgentTriggerDelivery(
+ envelope({ input: 'x'.repeat(MAX_AGENT_TRIGGER_ENVELOPE_BYTES) }),
+ ),
+ ).toThrow(`Agent trigger envelope exceeds ${MAX_AGENT_TRIGGER_ENVELOPE_BYTES} bytes`);
+ });
+});
diff --git a/packages/api/src/agents/triggers/delivery.ts b/packages/api/src/agents/triggers/delivery.ts
new file mode 100644
index 0000000000..5d511b263e
--- /dev/null
+++ b/packages/api/src/agents/triggers/delivery.ts
@@ -0,0 +1,115 @@
+import { createHash } from 'node:crypto';
+import type { AgentTriggerEnvelope } from './envelope';
+import { getAgentTriggerIdempotencyKey, parseAgentTriggerEnvelope } from './envelope';
+
+export const MAX_AGENT_TRIGGER_ENVELOPE_BYTES: number = 1024 * 1024;
+
+export interface AgentTriggerEnqueueOptions {
+ /** Trusted lane override. Matching lanes dispatch strictly in enqueue order. */
+ orderingKey?: string;
+ /** Delays first eligibility without changing source occurrence time. */
+ availableAt?: Date;
+}
+
+export interface PreparedAgentTriggerDelivery {
+ deliveryKey: string;
+ fingerprint: string;
+ orderingKey: string;
+ envelope: AgentTriggerEnvelope;
+ user: string;
+ tenantId?: string;
+ availableAt: Date;
+}
+
+export class AgentTriggerDeliveryError extends TypeError {
+ constructor(message: string) {
+ super(message);
+ this.name = 'AgentTriggerDeliveryError';
+ }
+}
+
+function canonicalJson(value: unknown): string {
+ if (value === null || typeof value === 'boolean' || typeof value === 'number') {
+ return JSON.stringify(value);
+ }
+ if (typeof value === 'string') {
+ return JSON.stringify(value);
+ }
+ if (Array.isArray(value)) {
+ return `[${value.map(canonicalJson).join(',')}]`;
+ }
+ const record = value as Record<string, unknown>;
+ return `{${Object.keys(record)
+ .sort()
+ .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
+ .join(',')}}`;
+}
+
+function digest(value: unknown): string {
+ return createHash('sha256').update(canonicalJson(value)).digest('hex');
+}
+
+function requireAvailableAt(value: Date | undefined): Date {
+ const availableAt = value ?? new Date();
+ if (!(availableAt instanceof Date) || !Number.isFinite(availableAt.getTime())) {
+ throw new AgentTriggerDeliveryError('availableAt must be a valid Date');
+ }
+ return new Date(availableAt);
+}
+
+function orderingIdentity(
+ envelope: AgentTriggerEnvelope,
+ configuredKey: string | undefined,
+): string {
+ let lane: unknown;
+ if (configuredKey != null) {
+ const trimmed = configuredKey.trim();
+ if (trimmed.length === 0 || trimmed.length > 256) {
+ throw new AgentTriggerDeliveryError('orderingKey must contain between 1 and 256 characters');
+ }
+ lane = ['explicit', trimmed];
+ } else {
+ lane = [
+ 'default',
+ envelope.event.source.type,
+ envelope.event.source.id,
+ envelope.mode,
+ envelope.target.agentId,
+ envelope.mode === 'steer' ? envelope.target.conversationId : '',
+ ];
+ }
+ return `trigger_lane_${digest([
+ envelope.principal.tenantId ?? '',
+ envelope.principal.userId,
+ lane,
+ ])}`;
+}
+
+/** Validates, bounds, detaches, and fingerprints one durable source delivery. */
+export function prepareAgentTriggerDelivery(
+ value: unknown,
+ options: AgentTriggerEnqueueOptions = {},
+): PreparedAgentTriggerDelivery {
+ const envelope = parseAgentTriggerEnvelope(value);
+ const serialized = canonicalJson(envelope);
+ const bytes = Buffer.byteLength(serialized, 'utf8');
+ if (bytes > MAX_AGENT_TRIGGER_ENVELOPE_BYTES) {
+ throw new AgentTriggerDeliveryError(
+ `Agent trigger envelope exceeds ${MAX_AGENT_TRIGGER_ENVELOPE_BYTES} bytes`,
+ );
+ }
+
+ // requestId and receivedAt identify an ingress attempt, not the logical
+ // source-event-to-target delivery. Excluding them lets a retried ingress use
+ // a fresh request trace without weakening content-conflict detection.
+ const { requestId: _requestId, receivedAt: _receivedAt, ...durableIdentity } = envelope;
+ return {
+ deliveryKey: getAgentTriggerIdempotencyKey(envelope),
+ fingerprint: digest(durableIdentity),
+ orderingKey: orderingIdentity(envelope, options.orderingKey),
+ envelope,
+ user: envelope.principal.userId,
+ ...(envelope.principal.tenantId != null && { tenantId: envelope.principal.tenantId }),
+ availableAt: requireAvailableAt(options.availableAt),
+ };
+}
diff --git a/packages/api/src/agents/triggers/engine.spec.ts b/packages/api/src/agents/triggers/engine.spec.ts
new file mode 100644
index 0000000000..4a0eaff1f5
--- /dev/null
+++ b/packages/api/src/agents/triggers/engine.spec.ts
@@ -0,0 +1,576 @@
+import type { AgentTriggerDeliveryRecord, AgentTriggerDeliveryStore } from './engine';
+import type { AgentTriggerExecutionResult } from './host';
+import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine';
+import { AgentTriggerDispatchError } from './dispatch';
+import { AgentTriggerExecutionError } from './host';
+
+jest.mock('@librechat/data-schemas', () => {
+ const actual = jest.requireActual('@librechat/data-schemas');
+ return {
+ ...actual,
+ logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+});
+
+const START = new Date('2026-08-17T12:00:00.000Z');
+
+const successResult = (): AgentTriggerExecutionResult => ({
+ mode: 'fire',
+ status: 'started',
+ conversationId: 'conversation-1',
+});
+
+function delivery(overrides: Partial<AgentTriggerDeliveryRecord> = {}): AgentTriggerDeliveryRecord {
+ return {
+ id: 'delivery-row-1',
+ user: 'user-1',
+ claimToken: 'claim-1',
+ deliveryKey: 'trigger_1',
+ fingerprint: 'fingerprint-1',
+ orderingKey: 'ordering-1',
+ laneSequence: 1,
+ envelope: { version: 1 },
+ status: 'leased',
+ attempts: 0,
+ availableAt: START,
+ createdAt: START,
+ leaseBy: 'worker-1',
+ leaseUntil: new Date(START.getTime() + 120_000),
+ ...overrides,
+ };
+}
+
+function storeWith(overrides: Partial<AgentTriggerDeliveryStore> = {}): AgentTriggerDeliveryStore {
+ return {
+ claimNext: jest.fn(async () => delivery()),
+ findEarlierUnsettled: jest.fn(async () => null),
+ release: jest.fn(async () => true),
+ beginAttempt: jest.fn(async () => 1),
+ defer: jest.fn(async () => true),
+ complete: jest.fn(async () => true),
+ retry: jest.fn(async () => true),
+ dead: jest.fn(async () => true),
+ ...overrides,
+ };
+}
+
+describe('createAgentTriggerDeliveryEngine', () => {
+ it('claims, dispatches, and completes with the per-claim fence', async () => {
+ const store = storeWith();
+ const dispatch = jest.fn(async () => successResult());
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START, workerId: 'worker-1' },
+ { concurrency: 1 },
+ );
+
+ await expect(engine.runTick()).resolves.toBe(1);
+
+ expect(dispatch).toHaveBeenCalledWith({ version: 1 }, { signal: expect.any(AbortSignal) });
+ expect(store.beginAttempt).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ });
+ expect(store.complete).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: 1,
+ result: successResult(),
+ settledAt: START,
+ });
+ });
+
+ it('cancels and drains an in-flight delivery for one user', async () => {
+ const store = storeWith();
+ let dispatchStarted: (() => void) | undefined;
+ const started = new Promise<void>((resolve) => {
+ dispatchStarted = resolve;
+ });
+ const dispatch = jest.fn(
+ async (_envelope: unknown, options?: { signal?: AbortSignal }) =>
+ new Promise<AgentTriggerExecutionResult>((_resolve, reject) => {
+ dispatchStarted?.();
+ options?.signal?.addEventListener(
+ 'abort',
+ () =>
+ reject(
+ new AgentTriggerExecutionError('cancelled', {
+ mode: 'fire',
+ certainty: 'ambiguous',
+ retryable: true,
+ code: 'ABORTED',
+ }),
+ ),
+ { once: true },
+ );
+ }),
+ );
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START, workerId: 'worker-1' },
+ { concurrency: 1, maxAttempts: 1 },
+ );
+
+ const tick = engine.runTick();
+ await started;
+ await engine.cancelUser('user-1');
+ await tick;
+
+ expect(store.defer).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: 1,
+ availableAt: new Date(START.getTime() + 5_000),
+ });
+ expect(store.retry).not.toHaveBeenCalled();
+ expect(store.dead).not.toHaveBeenCalled();
+ expect(store.complete).not.toHaveBeenCalled();
+ });
+
+ it('releases a claimed delivery without dispatch after its user is fenced', async () => {
+ const store = storeWith();
+ const dispatch = jest.fn(async () => successResult());
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START, workerId: 'worker-1' },
+ { concurrency: 1 },
+ );
+
+ await engine.cancelUser('user-1');
+ await expect(engine.runTick()).resolves.toBe(1);
+
+ expect(store.release).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ availableAt: START,
+ });
+ expect(store.beginAttempt).not.toHaveBeenCalled();
+ expect(dispatch).not.toHaveBeenCalled();
+
+ engine.releaseUserCancellation('user-1');
+ await engine.runTick();
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ });
+
+ it('honors Retry-After for a retryable admission rejection', async () => {
+ const store = storeWith();
+ const error = new AgentTriggerExecutionError('busy', {
+ mode: 'fire',
+ certainty: 'definite',
+ retryable: true,
+ code: 'RATE_LIMITED',
+ status: 429,
+ retryAfter: '120',
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: async () => Promise.reject(error), now: () => START },
+ { concurrency: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.retry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ claimToken: 'claim-1',
+ availableAt: new Date(START.getTime() + 120_000),
+ error: expect.objectContaining({
+ code: 'RATE_LIMITED',
+ certainty: 'definite',
+ retryable: true,
+ status: 429,
+ }),
+ }),
+ );
+ expect(store.dead).not.toHaveBeenCalled();
+ });
+
+ it('defers a fenced principal without consuming its delivery attempt', async () => {
+ const store = storeWith();
+ const engine = createAgentTriggerDeliveryEngine(
+ {
+ store,
+ dispatch: async () =>
+ Promise.reject(new AgentTriggerDeliveryDeferredError('principal fenced')),
+ now: () => START,
+ workerId: 'worker-1',
+ },
+ { concurrency: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.defer).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: 1,
+ availableAt: new Date(START.getTime() + 5_000),
+ });
+ expect(store.retry).not.toHaveBeenCalled();
+ expect(store.dead).not.toHaveBeenCalled();
+ });
+
+ it('defers a cross-replica account-deletion rejection without consuming its attempt', async () => {
+ const store = storeWith();
+ const engine = createAgentTriggerDeliveryEngine(
+ {
+ store,
+ dispatch: async () =>
+ Promise.reject(
+ new AgentTriggerExecutionError('account deletion is in progress', {
+ mode: 'fire',
+ certainty: 'definite',
+ retryable: false,
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ status: 409,
+ }),
+ ),
+ now: () => START,
+ workerId: 'worker-1',
+ },
+ { concurrency: 1, maxAttempts: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.defer).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: 1,
+ availableAt: new Date(START.getTime() + 5_000),
+ });
+ expect(store.retry).not.toHaveBeenCalled();
+ expect(store.dead).not.toHaveBeenCalled();
+ });
+
+ it('defers an auth-stage account-deletion rejection without consuming its attempt', async () => {
+ const store = storeWith();
+ const engine = createAgentTriggerDeliveryEngine(
+ {
+ store,
+ dispatch: async () =>
+ Promise.reject(
+ new AgentTriggerExecutionError('account deletion is in progress', {
+ mode: 'fire',
+ certainty: 'definite',
+ retryable: false,
+ code: 'ACCOUNT_DELETION_IN_PROGRESS',
+ status: 401,
+ }),
+ ),
+ now: () => START,
+ workerId: 'worker-1',
+ },
+ { concurrency: 1, maxAttempts: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.defer).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: 1,
+ availableAt: new Date(START.getTime() + 5_000),
+ });
+ expect(store.retry).not.toHaveBeenCalled();
+ expect(store.dead).not.toHaveBeenCalled();
+ });
+
+ it('does not shorten Retry-After to the exponential backoff cap', async () => {
+ const store = storeWith();
+ const error = new AgentTriggerExecutionError('maintenance', {
+ mode: 'fire',
+ certainty: 'definite',
+ retryable: true,
+ retryAfter: '3600',
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: async () => Promise.reject(error), now: () => START },
+ { concurrency: 1, retryCapMs: 1_000 },
+ );
+
+ await engine.runTick();
+
+ expect(store.retry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ availableAt: new Date(START.getTime() + 60 * 60_000),
+ }),
+ );
+ });
+
+ it('dead-letters invalid envelopes without retrying', async () => {
+ const store = storeWith();
+ const engine = createAgentTriggerDeliveryEngine(
+ {
+ store,
+ dispatch: async () => Promise.reject(new AgentTriggerDispatchError('invalid envelope')),
+ now: () => START,
+ },
+ { concurrency: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.dead).toHaveBeenCalledWith(
+ expect.objectContaining({
+ claimToken: 'claim-1',
+ error: expect.objectContaining({
+ code: 'INVALID_ENVELOPE',
+ certainty: 'definite',
+ retryable: false,
+ }),
+ }),
+ );
+ expect(store.retry).not.toHaveBeenCalled();
+ });
+
+ it('dead-letters an exhausted row without dispatching again', async () => {
+ const store = storeWith({
+ claimNext: jest.fn(async () => delivery({ attempts: 8 })),
+ });
+ const dispatch = jest.fn(async () => successResult());
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START },
+ { concurrency: 1, maxAttempts: 8 },
+ );
+
+ await engine.runTick();
+
+ expect(dispatch).not.toHaveBeenCalled();
+ expect(store.beginAttempt).not.toHaveBeenCalled();
+ expect(store.dead).toHaveBeenCalledWith(expect.objectContaining({ claimToken: 'claim-1' }));
+ });
+
+ it('rechecks a leased predecessor promptly instead of waiting for its full lease', async () => {
+ const store = storeWith({
+ findEarlierUnsettled: jest.fn(async () => ({
+ availableAt: new Date(START.getTime() + 2_000),
+ leaseUntil: new Date(START.getTime() + 4_000),
+ })),
+ });
+ const dispatch = jest.fn(async () => successResult());
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START },
+ { concurrency: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.release).toHaveBeenCalledWith({
+ id: 'delivery-row-1',
+ workerId: expect.any(String),
+ claimToken: 'claim-1',
+ availableAt: new Date(START.getTime() + 250),
+ });
+ expect(store.beginAttempt).not.toHaveBeenCalled();
+ expect(dispatch).not.toHaveBeenCalled();
+ });
+
+ it('does not poll ahead of a pending predecessor eligibility time', async () => {
+ const store = storeWith({
+ findEarlierUnsettled: jest.fn(async () => ({
+ availableAt: new Date(START.getTime() + 4_000),
+ })),
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: async () => successResult(), now: () => START },
+ { concurrency: 1 },
+ );
+
+ await engine.runTick();
+
+ expect(store.release).toHaveBeenCalledWith(
+ expect.objectContaining({ availableAt: new Date(START.getTime() + 4_000) }),
+ );
+ });
+
+ it('starts independent deliveries up to the configured concurrency', async () => {
+ let releaseDispatch: (() => void) | undefined;
+ const gate = new Promise<void>((resolve) => {
+ releaseDispatch = resolve;
+ });
+ let dispatchStarts = 0;
+ let bothStarted: (() => void) | undefined;
+ const started = new Promise<void>((resolve) => {
+ bothStarted = resolve;
+ });
+ const records = [delivery(), delivery({ id: 'delivery-row-2', claimToken: 'claim-2' })];
+ const store = storeWith({
+ claimNext: jest.fn(async () => records.shift() ?? null),
+ });
+ const dispatch = jest.fn(async () => {
+ dispatchStarts += 1;
+ if (dispatchStarts === 2) {
+ bothStarted?.();
+ }
+ await gate;
+ return successResult();
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START },
+ { concurrency: 2 },
+ );
+
+ const tick = engine.runTick();
+ await started;
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ releaseDispatch?.();
+ await tick;
+ });
+
+ it('coalesces overlapping ticks into one claim pass', async () => {
+ let releaseClaim: (() => void) | undefined;
+ const gate = new Promise<void>((resolve) => {
+ releaseClaim = resolve;
+ });
+ const claimNext = jest.fn(async () => {
+ await gate;
+ return null;
+ });
+ const store = storeWith({ claimNext });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: async () => successResult() },
+ { concurrency: 1 },
+ );
+
+ const first = engine.runTick();
+ const second = engine.runTick();
+ releaseClaim?.();
+ await Promise.all([first, second]);
+ expect(claimNext).toHaveBeenCalledTimes(1);
+ });
+
+ it('uses one claim when an idle worker has no due work', async () => {
+ const claimNext = jest.fn(async () => null);
+ const engine = createAgentTriggerDeliveryEngine(
+ { store: storeWith({ claimNext }), dispatch: async () => successResult() },
+ { concurrency: 4 },
+ );
+
+ await expect(engine.runTick()).resolves.toBe(0);
+
+ expect(claimNext).toHaveBeenCalledTimes(1);
+ });
+
+ it('fills an open worker slot while an earlier independent delivery is still running', async () => {
+ const records = [delivery({ envelope: { delivery: 1 } })];
+ const store = storeWith({
+ claimNext: jest.fn(async () => records.shift() ?? null),
+ });
+ let releaseFirst: (() => void) | undefined;
+ const firstGate = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+ let releaseSecond: (() => void) | undefined;
+ const secondGate = new Promise<void>((resolve) => {
+ releaseSecond = resolve;
+ });
+ let firstStarted: (() => void) | undefined;
+ const firstStart = new Promise<void>((resolve) => {
+ firstStarted = resolve;
+ });
+ let secondStarted: (() => void) | undefined;
+ const secondStart = new Promise<void>((resolve) => {
+ secondStarted = resolve;
+ });
+ const dispatch = jest.fn(async (value: unknown) => {
+ const deliveryNumber = (value as { delivery: number }).delivery;
+ if (deliveryNumber === 1) {
+ firstStarted?.();
+ await firstGate;
+ } else {
+ secondStarted?.();
+ await secondGate;
+ }
+ return successResult();
+ });
+ const engine = createAgentTriggerDeliveryEngine({ store, dispatch }, { concurrency: 2 });
+
+ engine.start();
+ await firstStart;
+ records.push(
+ delivery({ id: 'delivery-row-2', claimToken: 'claim-2', envelope: { delivery: 2 } }),
+ );
+ engine.wake();
+ await secondStart;
+
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ releaseFirst?.();
+ releaseSecond?.();
+ await engine.stop();
+ });
+
+ it('retries ambiguously when execution succeeds but result persistence fails', async () => {
+ const store = storeWith({
+ complete: jest.fn(async () => Promise.reject(new Error('mongo unavailable'))),
+ });
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch: async () => successResult(), now: () => START, random: () => 0 },
+ { concurrency: 1, retryBaseMs: 1_000 },
+ );
+
+ await engine.runTick();
+
+ expect(store.retry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ claimToken: 'claim-1',
+ availableAt: new Date(START.getTime() + 500),
+ error: expect.objectContaining({
+ code: 'RESULT_PERSISTENCE_FAILED',
+ certainty: 'ambiguous',
+ retryable: true,
+ }),
+ }),
+ );
+ });
+
+ it('aborts in-flight dispatch during stop and persists a retry', async () => {
+ const store = storeWith();
+ let dispatchStarted: (() => void) | undefined;
+ const started = new Promise<void>((resolve) => {
+ dispatchStarted = resolve;
+ });
+ const dispatch = jest.fn(
+ (
+ _envelope: unknown,
+ options?: { signal?: AbortSignal },
+ ): Promise<AgentTriggerExecutionResult> =>
+ new Promise((_, reject) => {
+ dispatchStarted?.();
+ options?.signal?.addEventListener(
+ 'abort',
+ () =>
+ reject(
+ new AgentTriggerExecutionError('stopping', {
+ mode: 'fire',
+ certainty: 'definite',
+ retryable: true,
+ code: 'ABORTED',
+ }),
+ ),
+ { once: true },
+ );
+ }),
+ );
+ const engine = createAgentTriggerDeliveryEngine(
+ { store, dispatch, now: () => START },
+ { concurrency: 1 },
+ );
+
+ const tick = engine.runTick();
+ await started;
+ await engine.stop();
+ await tick;
+
+ expect(store.retry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ claimToken: 'claim-1',
+ error: expect.objectContaining({ code: 'ABORTED', retryable: true }),
+ }),
+ );
+ });
+});
diff --git a/packages/api/src/agents/triggers/engine.ts b/packages/api/src/agents/triggers/engine.ts
new file mode 100644
index 0000000000..eb3e0a42a9
--- /dev/null
+++ b/packages/api/src/agents/triggers/engine.ts
@@ -0,0 +1,590 @@
+import { randomUUID } from 'node:crypto';
+import { logger, runAsSystem } from '@librechat/data-schemas';
+import type { AgentTriggerExecutionResult } from './host';
+import { AgentTriggerDispatchError } from './dispatch';
+import { AgentTriggerExecutionError } from './host';
+
+const DEFAULT_CONCURRENCY = 4;
+const DEFAULT_LEASE_MS = 2 * 60_000;
+const DEFAULT_MAX_ATTEMPTS = 8;
+const DEFAULT_RETRY_BASE_MS = 1_000;
+const DEFAULT_RETRY_CAP_MS = 5 * 60_000;
+const DEFAULT_TICK_MS = 1_000;
+const ORDERING_RECHECK_MS = 250;
+const DEFAULT_DEFER_MS = 5_000;
+const MAX_RETRY_AFTER_MS = 24 * 60 * 60_000;
+
+/** A pre-dispatch condition that must not consume the delivery's retry budget. */
+export class AgentTriggerDeliveryDeferredError extends Error {
+ readonly delayMs: number;
+
+ constructor(message: string, delayMs: number = DEFAULT_DEFER_MS) {
+ super(message);
+ this.name = 'AgentTriggerDeliveryDeferredError';
+ this.delayMs = positiveInteger(delayMs, DEFAULT_DEFER_MS, 'delayMs');
+ }
+}
+
+export type AgentTriggerDeliveryStatus = 'staging' | 'pending' | 'leased' | 'succeeded' | 'dead';
+
+export interface AgentTriggerDeliveryFailure {
+ code: string;
+ message: string;
+ certainty: 'definite' | 'ambiguous';
+ retryable: boolean;
+ attemptedAt: Date;
+ status?: number;
+}
+
+export interface AgentTriggerDeliveryRecord {
+ id: string;
+ user: string;
+ claimToken: string;
+ deliveryKey: string;
+ fingerprint: string;
+ orderingKey: string;
+ laneSequence: number;
+ envelope: unknown;
+ status: AgentTriggerDeliveryStatus;
+ attempts: number;
+ availableAt: Date;
+ createdAt: Date;
+ leaseBy?: string;
+ leaseUntil?: Date;
+ lastError?: AgentTriggerDeliveryFailure;
+}
+
+export interface AgentTriggerOrderingBlock {
+ availableAt: Date;
+ leaseUntil?: Date;
+}
+
+export interface AgentTriggerDeliveryStore {
+ claimNext: (input: {
+ workerId: string;
+ claimToken: string;
+ now: Date;
+ leaseUntil: Date;
+ }) => Promise<AgentTriggerDeliveryRecord | null>;
+ findEarlierUnsettled: (
+ delivery: AgentTriggerDeliveryRecord,
+ ) => Promise<AgentTriggerOrderingBlock | null>;
+ release: (input: {
+ id: string;
+ workerId: string;
+ claimToken: string;
+ availableAt: Date;
+ }) => Promise<boolean>;
+ beginAttempt: (input: {
+ id: string;
+ workerId: string;
+ claimToken: string;
+ now: Date;
+ }) => Promise<number | null>;
+ defer: (input: {
+ id: string;
+ workerId: string;
+ claimToken: string;
+ attempt: number;
+ availableAt: Date;
+ }) => Promise<boolean>;
+ complete: (input: {
+ id: string;
+ workerId: string;
+ claimToken: string;
+ attempt: number;
+ result: AgentTriggerExecutionResult;
+ settledAt: Date;
+ }) => Promise<boolean>;
+ retry: (input: {
+ id: string;
+ workerId: string;
+ claimToken: string;
+ attempt: number;
+ error: AgentTriggerDeliveryFailure;
+ availableAt: Date;
+ }) => Promise<boolean>;
+ dead: (input: {
+ id: string;
+ workerId: string;
+ claimToken: string;
+ attempt: number;
+ error: AgentTriggerDeliveryFailure;
+ settledAt: Date;
+ }) => Promise<boolean>;
+}
+
+export interface AgentTriggerDeliveryEngineOptions {
+ concurrency?: number;
+ leaseMs?: number;
+ maxAttempts?: number;
+ retryBaseMs?: number;
+ retryCapMs?: number;
+ tickMs?: number;
+}
+
+export interface AgentTriggerDeliveryEngineDeps {
+ store: AgentTriggerDeliveryStore;
+ dispatch: (
+ envelope: unknown,
+ options?: { signal?: AbortSignal },
+ ) => Promise<AgentTriggerExecutionResult>;
+ now?: () => Date;
+ random?: () => number;
+ workerId?: string;
+}
+
+export interface AgentTriggerDeliveryEngine {
+ start: () => void;
+ stop: () => Promise<void>;
+ cancelUser: (userId: string) => Promise<void>;
+ releaseUserCancellation: (userId: string) => void;
+ wake: () => void;
+ runTick: () => Promise<number>;
+}
+
+function positiveInteger(value: number | undefined, fallback: number, name: string): number {
+ const resolved = value ?? fallback;
+ if (!Number.isSafeInteger(resolved) || resolved <= 0) {
+ throw new TypeError(`${name} must be a positive integer`);
+ }
+ return resolved;
+}
+
+function failure(error: unknown, attemptedAt: Date): AgentTriggerDeliveryFailure {
+ if (error instanceof AgentTriggerExecutionError) {
+ return {
+ code: error.code ?? 'DELIVERY_REJECTED',
+ message: error.message,
+ certainty: error.certainty,
+ retryable: error.retryable,
+ attemptedAt,
+ ...(error.status != null && { status: error.status }),
+ };
+ }
+ if (error instanceof AgentTriggerDispatchError) {
+ return {
+ code: 'INVALID_ENVELOPE',
+ message: error.message,
+ certainty: 'definite',
+ retryable: false,
+ attemptedAt,
+ };
+ }
+ return {
+ code: 'DELIVERY_FAILED',
+ message: error instanceof Error ? error.message : String(error),
+ certainty: 'definite',
+ retryable: true,
+ attemptedAt,
+ };
+}
+
+function retryAt(
+ error: unknown,
+ attempt: number,
+ now: Date,
+ baseMs: number,
+ capMs: number,
+ random: () => number,
+): Date {
+ if (error instanceof AgentTriggerExecutionError && error.retryAfter != null) {
+ const seconds = Number(error.retryAfter);
+ if (Number.isFinite(seconds) && seconds >= 0) {
+ return new Date(now.getTime() + Math.min(seconds * 1_000, MAX_RETRY_AFTER_MS));
+ }
+ const absolute = Date.parse(error.retryAfter);
+ if (Number.isFinite(absolute) && absolute > now.getTime()) {
+ return new Date(Math.min(absolute, now.getTime() + MAX_RETRY_AFTER_MS));
+ }
+ }
+ const exponent = Math.min(attempt - 1, 30);
+ const delay = Math.min(baseMs * 2 ** exponent, capMs);
+ return new Date(now.getTime() + Math.floor(delay / 2 + random() * (delay / 2)));
+}
+
+function isAccountDeletionDeferral(error: unknown): boolean {
+ return (
+ error instanceof AgentTriggerExecutionError &&
+ error.code === 'ACCOUNT_DELETION_IN_PROGRESS' &&
+ (error.status === 401 || error.status === 409)
+ );
+}
+
+/** Durable, lease-fenced delivery runner shared by every trusted event source. */
+export function createAgentTriggerDeliveryEngine(
+ deps: AgentTriggerDeliveryEngineDeps,
+ options: AgentTriggerDeliveryEngineOptions = {},
+): AgentTriggerDeliveryEngine {
+ const concurrency = positiveInteger(options.concurrency, DEFAULT_CONCURRENCY, 'concurrency');
+ const leaseMs = positiveInteger(options.leaseMs, DEFAULT_LEASE_MS, 'leaseMs');
+ const maxAttempts = positiveInteger(options.maxAttempts, DEFAULT_MAX_ATTEMPTS, 'maxAttempts');
+ const retryBaseMs = positiveInteger(options.retryBaseMs, DEFAULT_RETRY_BASE_MS, 'retryBaseMs');
+ const retryCapMs = positiveInteger(options.retryCapMs, DEFAULT_RETRY_CAP_MS, 'retryCapMs');
+ const tickMs = positiveInteger(options.tickMs, DEFAULT_TICK_MS, 'tickMs');
+ const now = deps.now ?? (() => new Date());
+ const random = deps.random ?? Math.random;
+ const workerId = deps.workerId ?? `${process.pid}-${randomUUID()}`;
+ const controllers = new Map<AbortController, string>();
+ const processing = new Set<Promise<void>>();
+ const processingByUser = new Map<string, Set<Promise<void>>>();
+ const cancelledUsers = new Set<string>();
+ let stopped = false;
+ let started = false;
+ let repumpRequested = false;
+ let timer: NodeJS.Timeout | undefined;
+ let activeClaim: Promise<{ count: number; processing: Promise<void>[] }> | undefined;
+
+ const processDelivery = async (delivery: AgentTriggerDeliveryRecord): Promise<void> => {
+ const userId = String(delivery.user);
+ if (cancelledUsers.has(userId)) {
+ await deps.store.release({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ availableAt: now(),
+ });
+ return;
+ }
+
+ const block = await deps.store.findEarlierUnsettled(delivery);
+ if (block != null) {
+ const recheckAt = now().getTime() + ORDERING_RECHECK_MS;
+ const nextCheck =
+ block.leaseUntil == null ? Math.max(recheckAt, block.availableAt.getTime()) : recheckAt;
+ await deps.store.release({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ availableAt: new Date(nextCheck),
+ });
+ return;
+ }
+
+ if (delivery.attempts >= maxAttempts) {
+ const recorded =
+ delivery.lastError ??
+ failure(new Error('Delivery attempt limit was already exhausted'), now());
+ const deadLettered = await deps.store.dead({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ attempt: delivery.attempts,
+ error: recorded,
+ settledAt: now(),
+ });
+ if (deadLettered) {
+ logger.error('[agent-triggers] delivery dead-lettered after exhausting retries', {
+ deliveryKey: delivery.deliveryKey,
+ attempts: delivery.attempts,
+ code: recorded.code,
+ });
+ }
+ return;
+ }
+
+ if (stopped) {
+ await deps.store.release({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ availableAt: now(),
+ });
+ return;
+ }
+
+ if (cancelledUsers.has(userId)) {
+ await deps.store.release({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ availableAt: now(),
+ });
+ return;
+ }
+
+ const attempt = await deps.store.beginAttempt({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ now: now(),
+ });
+ if (attempt == null) {
+ return;
+ }
+
+ const controller = new AbortController();
+ controllers.set(controller, userId);
+ if (stopped || cancelledUsers.has(userId)) {
+ controller.abort(new Error('Agent trigger delivery engine is stopping'));
+ }
+ try {
+ let result: AgentTriggerExecutionResult;
+ try {
+ result = await deps.dispatch(delivery.envelope, { signal: controller.signal });
+ } catch (error) {
+ const attemptedAt = now();
+ const deletionCancelled = controller.signal.aborted && cancelledUsers.has(userId);
+ const deletionRejected = isAccountDeletionDeferral(error);
+ if (
+ error instanceof AgentTriggerDeliveryDeferredError ||
+ deletionCancelled ||
+ deletionRejected
+ ) {
+ const delayMs =
+ error instanceof AgentTriggerDeliveryDeferredError ? error.delayMs : DEFAULT_DEFER_MS;
+ const availableAt = new Date(attemptedAt.getTime() + delayMs);
+ const deferred = await deps.store.defer({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ attempt,
+ availableAt,
+ });
+ if (deferred) {
+ logger.info('[agent-triggers] delivery deferred without consuming an attempt', {
+ deliveryKey: delivery.deliveryKey,
+ reason: deletionCancelled || deletionRejected ? 'account_deletion' : 'pre_dispatch',
+ availableAt: availableAt.toISOString(),
+ });
+ }
+ return;
+ }
+ const recorded = failure(error, attemptedAt);
+ if (!recorded.retryable || attempt >= maxAttempts) {
+ const deadLettered = await deps.store.dead({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ attempt,
+ error: recorded,
+ settledAt: attemptedAt,
+ });
+ if (deadLettered) {
+ logger.error('[agent-triggers] delivery dead-lettered', {
+ deliveryKey: delivery.deliveryKey,
+ attempt,
+ code: recorded.code,
+ certainty: recorded.certainty,
+ });
+ }
+ return;
+ }
+ const availableAt = retryAt(error, attempt, attemptedAt, retryBaseMs, retryCapMs, random);
+ const retrying = await deps.store.retry({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ attempt,
+ error: recorded,
+ availableAt,
+ });
+ if (retrying) {
+ logger.warn('[agent-triggers] delivery scheduled for retry', {
+ deliveryKey: delivery.deliveryKey,
+ attempt,
+ code: recorded.code,
+ certainty: recorded.certainty,
+ availableAt: availableAt.toISOString(),
+ });
+ }
+ return;
+ }
+
+ const settledAt = now();
+ try {
+ await deps.store.complete({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ attempt,
+ result,
+ settledAt,
+ });
+ } catch (error) {
+ const recorded: AgentTriggerDeliveryFailure = {
+ code: 'RESULT_PERSISTENCE_FAILED',
+ message: error instanceof Error ? error.message : String(error),
+ certainty: 'ambiguous',
+ retryable: true,
+ attemptedAt: settledAt,
+ };
+ const availableAt = retryAt(error, attempt, settledAt, retryBaseMs, retryCapMs, random);
+ const retrying = await deps.store.retry({
+ id: delivery.id,
+ workerId,
+ claimToken: delivery.claimToken,
+ attempt,
+ error: recorded,
+ availableAt,
+ });
+ if (retrying) {
+ logger.warn(
+ '[agent-triggers] accepted delivery result could not be persisted; retrying',
+ {
+ deliveryKey: delivery.deliveryKey,
+ attempt,
+ availableAt: availableAt.toISOString(),
+ },
+ );
+ }
+ }
+ } finally {
+ controllers.delete(controller);
+ }
+ };
+
+ const claimOne = (): Promise<AgentTriggerDeliveryRecord | null> => {
+ const claimedAt = now();
+ return deps.store.claimNext({
+ workerId,
+ claimToken: randomUUID(),
+ now: claimedAt,
+ leaseUntil: new Date(claimedAt.getTime() + leaseMs),
+ });
+ };
+
+ const runClaimPass = async (): Promise<{ count: number; processing: Promise<void>[] }> => {
+ if (stopped) {
+ return { count: 0, processing: [] };
+ }
+ const openSlots = concurrency - processing.size;
+ if (openSlots <= 0) {
+ return { count: 0, processing: [] };
+ }
+
+ const deliveries: AgentTriggerDeliveryRecord[] = [];
+ try {
+ const first = await claimOne();
+ if (first == null) {
+ return { count: 0, processing: [] };
+ }
+ deliveries.push(first);
+ } catch (error) {
+ logger.error('[agent-triggers] delivery claim failed:', error);
+ return { count: 0, processing: [] };
+ }
+
+ if (openSlots > 1) {
+ const claimed = await Promise.allSettled(
+ Array.from({ length: openSlots - 1 }, () => claimOne()),
+ );
+ for (const result of claimed) {
+ if (result.status === 'fulfilled' && result.value != null) {
+ deliveries.push(result.value);
+ } else if (result.status === 'rejected') {
+ logger.error('[agent-triggers] delivery claim failed:', result.reason);
+ }
+ }
+ }
+
+ const batch: Promise<void>[] = [];
+ for (const delivery of deliveries) {
+ const userId = String(delivery.user);
+ const task = runAsSystem(() => processDelivery(delivery)).catch((error) => {
+ logger.error('[agent-triggers] delivery processing failed:', error);
+ });
+ const tracked = task.finally(() => {
+ processing.delete(tracked);
+ const userProcessing = processingByUser.get(userId);
+ userProcessing?.delete(tracked);
+ if (userProcessing?.size === 0) {
+ processingByUser.delete(userId);
+ }
+ if (started && !stopped) {
+ queueMicrotask(wake);
+ }
+ });
+ processing.add(tracked);
+ const userProcessing = processingByUser.get(userId) ?? new Set<Promise<void>>();
+ userProcessing.add(tracked);
+ processingByUser.set(userId, userProcessing);
+ batch.push(tracked);
+ }
+ return { count: deliveries.length, processing: batch };
+ };
+
+ const claimAvailable = (): Promise<{ count: number; processing: Promise<void>[] }> => {
+ if (activeClaim != null) {
+ return activeClaim;
+ }
+ activeClaim = runAsSystem(runClaimPass).finally(() => {
+ activeClaim = undefined;
+ if (repumpRequested && !stopped) {
+ repumpRequested = false;
+ queueMicrotask(wake);
+ }
+ });
+ return activeClaim;
+ };
+
+ const runTick = async (): Promise<number> => {
+ const batch = await claimAvailable();
+ await Promise.allSettled(batch.processing);
+ return batch.count;
+ };
+
+ function wake(): void {
+ if (stopped) {
+ return;
+ }
+ if (activeClaim != null) {
+ repumpRequested = true;
+ return;
+ }
+ void claimAvailable().catch((error) =>
+ logger.error('[agent-triggers] delivery claim pass failed:', error),
+ );
+ }
+
+ const schedule = () => {
+ if (stopped) {
+ return;
+ }
+ timer = setTimeout(() => {
+ wake();
+ schedule();
+ }, tickMs);
+ timer.unref();
+ };
+
+ return {
+ start: () => {
+ if (started || stopped) {
+ return;
+ }
+ started = true;
+ wake();
+ schedule();
+ },
+ stop: async () => {
+ stopped = true;
+ if (timer != null) {
+ clearTimeout(timer);
+ }
+ for (const controller of controllers.keys()) {
+ controller.abort();
+ }
+ await activeClaim?.catch(() => undefined);
+ await Promise.allSettled([...processing]);
+ },
+ cancelUser: async (userId) => {
+ cancelledUsers.add(userId);
+ for (const [controller, activeUserId] of controllers) {
+ if (activeUserId === userId) {
+ controller.abort(new Error('Agent trigger delivery cancelled for account deletion'));
+ }
+ }
+ await activeClaim?.catch(() => undefined);
+ const userProcessing = processingByUser.get(userId);
+ if (userProcessing != null) {
+ await Promise.allSettled([...userProcessing]);
+ }
+ },
+ releaseUserCancellation: (userId) => {
+ cancelledUsers.delete(userId);
+ },
+ wake,
+ runTick,
+ };
+}
diff --git a/packages/api/src/agents/triggers/host.spec.ts b/packages/api/src/agents/triggers/host.spec.ts
index d94a935b05..86f4ca6f70 100644
--- a/packages/api/src/agents/triggers/host.spec.ts
+++ b/packages/api/src/agents/triggers/host.spec.ts
@@ -120,6 +120,7 @@ describe('createAgentTriggerExecutionHost fire adapter', () => {
expect(String(input)).toBe('https://chat.example.test/base/api/agents/chat/agents');
const headers = new Headers(init?.headers);
expect(headers.get('authorization')).toBe('Bearer signed-token');
+ expect(headers.get('x-lc-agent-trigger')).toBe('1');
expect(headers.get('x-request-id')).toBe(idempotencyKey);
expect(headers.get('x-librechat-generation-protocol')).toBe('2');
expect(headers.get('user-agent')).toContain('LibreChat-Agent-Trigger/1');
@@ -499,6 +500,7 @@ describe('createAgentTriggerExecutionHost steer adapter', () => {
expect(String(input)).toBe('https://chat.example.test/base/api/agents/chat/steer/deliver');
const headers = new Headers(init?.headers);
expect(headers.get('authorization')).toBe('Bearer signed-token');
+ expect(headers.get('x-lc-agent-trigger')).toBe('1');
expect(headers.get('x-request-id')).toBe(idempotencyKey);
expect(headers.get('x-librechat-generation-protocol')).toBe('2');
expect(JSON.parse(String(init?.body))).toEqual({
diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts
index a59be869b2..730a75c10d 100644
--- a/packages/api/src/agents/triggers/host.ts
+++ b/packages/api/src/agents/triggers/host.ts
@@ -467,6 +467,7 @@ async function fire(
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
'User-Agent': TRIGGER_USER_AGENT,
+ 'x-lc-agent-trigger': '1',
'x-request-id': context.idempotencyKey,
[GENERATION_PROTOCOL_HEADER]: '2',
},
@@ -650,6 +651,7 @@ async function steer(
Authorization: `Bearer ${token}`,
'Content-Type': 'application/json',
'User-Agent': TRIGGER_USER_AGENT,
+ 'x-lc-agent-trigger': '1',
'x-request-id': context.idempotencyKey,
[GENERATION_PROTOCOL_HEADER]: '2',
},
diff --git a/packages/api/src/agents/triggers/index.ts b/packages/api/src/agents/triggers/index.ts
index 3c75f2ba09..ab2c692942 100644
--- a/packages/api/src/agents/triggers/index.ts
+++ b/packages/api/src/agents/triggers/index.ts
@@ -1,3 +1,6 @@
export * from './dispatch';
+export * from './delivery';
export * from './envelope';
export * from './host';
+export * from './service';
+export * from './engine';
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
new file mode 100644
index 0000000000..f9e0f7a329
--- /dev/null
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -0,0 +1,332 @@
+import { getTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas';
+import type { AgentTriggerDeliveryPersistence, AgentTriggerStoredRecord } from './service';
+import { AgentTriggerServiceUnavailableError, createAgentTriggerService } from './service';
+import { __resetShutdownStateForTests } from '../../app/shutdown';
+import { createAgentTriggerEnvelope } from './envelope';
+
+jest.mock('@librechat/data-schemas', () => {
+ const actual = jest.requireActual('@librechat/data-schemas');
+ return {
+ ...actual,
+ logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+});
+
+const START = new Date('2026-08-17T12:00:00.000Z');
+
+const envelope = () =>
+ createAgentTriggerEnvelope({
+ mode: 'fire',
+ requestId: 'request-1',
+ deliveryId: 'delivery-1',
+ receivedAt: 20,
+ principal: { id: '507f1f77bcf86cd799439011', tenantId: 'tenant-1' },
+ target: { agentId: 'agent-1' },
+ event: {
+ id: 'event-1',
+ type: 'resource.ready',
+ occurredAt: 10,
+ source: { id: 'source-1', type: 'webhook' },
+ },
+ input: 'Handle the ready resource.',
+ });
+
+function deliveryRecord(overrides: Partial<AgentTriggerStoredRecord> = {}) {
+ return {
+ id: 'delivery-row-1',
+ user: '507f1f77bcf86cd799439011',
+ deliveryKey: 'trigger_1',
+ fingerprint: 'fingerprint-1',
+ orderingKey: 'ordering-1',
+ laneSequence: 1,
+ envelope: envelope(),
+ tenantId: 'tenant-1',
+ status: 'pending' as const,
+ attempts: 0,
+ availableAt: START,
+ createdAt: START,
+ ...overrides,
+ };
+}
+
+function deliveryMethods(
+ overrides: Partial<AgentTriggerDeliveryPersistence> = {},
+): AgentTriggerDeliveryPersistence {
+ return {
+ ensureAgentTriggerDeliveryIndexes: jest.fn(async () => undefined),
+ enqueueAgentTriggerDelivery: jest.fn(async (input) => ({
+ delivery: deliveryRecord({
+ deliveryKey: input.deliveryKey,
+ fingerprint: input.fingerprint,
+ orderingKey: input.orderingKey,
+ laneSequence: 1,
+ envelope: input.envelope,
+ availableAt: input.availableAt,
+ }),
+ replayed: false,
+ })),
+ claimNextAgentTriggerDelivery: jest.fn(async () => null),
+ findEarlierAgentTriggerDelivery: jest.fn(async () => null),
+ releaseAgentTriggerDelivery: jest.fn(async () => true),
+ beginAgentTriggerDeliveryAttempt: jest.fn(async () => 1),
+ deferAgentTriggerDeliveryAttempt: jest.fn(async () => true),
+ completeAgentTriggerDelivery: jest.fn(async () => true),
+ retryAgentTriggerDelivery: jest.fn(async () => true),
+ deadLetterAgentTriggerDelivery: jest.fn(async () => true),
+ getAgentTriggerDelivery: jest.fn(async () => null),
+ getAgentTriggerDeadLetters: jest.fn(async () => []),
+ requeueAgentTriggerDelivery: jest.fn(async () => null),
+ countActiveAgentTriggerDeliveriesByUser: jest.fn(async () => 0),
+ recoverAgentTriggerLanePublications: jest.fn(async () => 0),
+ reclaimInactiveAgentTriggerLanes: jest.fn(async () => 0),
+ prepareAgentTriggerUserPurge: jest.fn(async () => undefined),
+ cancelAgentTriggerUserPurge: jest.fn(async () => true),
+ recoverAgentTriggerUserPurges: jest.fn(async () => 0),
+ deleteAgentTriggerDeliveriesByUser: jest.fn(async () => undefined),
+ ...overrides,
+ };
+}
+
+describe('durable agent trigger service', () => {
+ const originalSelfUrl = process.env.AGENT_TRIGGERS_SELF_URL;
+
+ beforeEach(() => {
+ __resetShutdownStateForTests();
+ delete process.env.AGENT_TRIGGERS_SELF_URL;
+ });
+
+ afterEach(() => {
+ __resetShutdownStateForTests();
+ });
+
+ afterAll(() => {
+ if (originalSelfUrl == null) {
+ delete process.env.AGENT_TRIGGERS_SELF_URL;
+ } else {
+ process.env.AGENT_TRIGGERS_SELF_URL = originalSelfUrl;
+ }
+ });
+
+ it('fails closed before indexes and the delivery engine are ready', async () => {
+ const service = createAgentTriggerService({ methods: deliveryMethods() });
+
+ await expect(service.enqueue(envelope())).rejects.toBeInstanceOf(
+ AgentTriggerServiceUnavailableError,
+ );
+ await service.stop();
+ });
+
+ it('refuses to arm without a reachable self origin', async () => {
+ const service = createAgentTriggerService({ methods: deliveryMethods() });
+
+ await expect(service.initialize()).rejects.toThrow(
+ 'requires a valid listener address or AGENT_TRIGGERS_SELF_URL',
+ );
+ await service.stop();
+ });
+
+ it('initializes indexes, enqueues normalized source-neutral work, and starts claims', async () => {
+ const methods = deliveryMethods();
+ const service = createAgentTriggerService({
+ methods,
+ mintToken: () => 'token',
+ fetch: async () => new Response('{}', { status: 500 }),
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+
+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
+ const receipt = await service.enqueue(envelope(), {
+ orderingKey: 'match-42',
+ availableAt: START,
+ });
+
+ expect(methods.ensureAgentTriggerDeliveryIndexes).toHaveBeenCalledTimes(1);
+ expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalled();
+ expect(methods.enqueueAgentTriggerDelivery).toHaveBeenCalledWith(
+ expect.objectContaining({
+ deliveryKey: expect.stringMatching(/^trigger_/),
+ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/),
+ orderingKey: expect.stringMatching(/^trigger_lane_/),
+ user: '507f1f77bcf86cd799439011',
+ tenantId: 'tenant-1',
+ availableAt: START,
+ }),
+ );
+ expect(receipt).toMatchObject({
+ id: 'delivery-row-1',
+ status: 'pending',
+ replayed: false,
+ availableAt: START,
+ });
+ await service.stop();
+ });
+
+ it('rejects enqueue before persistence when the principal was deleted', async () => {
+ const methods = deliveryMethods();
+ const service = createAgentTriggerService({
+ methods,
+ isPrincipalActive: async () => false,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+
+ await expect(service.enqueue(envelope())).rejects.toThrow(
+ 'Agent trigger delivery principal is no longer active',
+ );
+ expect(methods.enqueueAgentTriggerDelivery).not.toHaveBeenCalled();
+ await service.stop();
+ });
+
+ it('preserves a just-enqueued row when deletion wins the admission race', async () => {
+ const methods = deliveryMethods();
+ const isPrincipalActive = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false);
+ const service = createAgentTriggerService({
+ methods,
+ isPrincipalActive,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+
+ await expect(service.enqueue(envelope())).rejects.toThrow(
+ 'Agent trigger delivery principal is no longer active',
+ );
+ expect(methods.countActiveAgentTriggerDeliveriesByUser).toHaveBeenCalled();
+ expect(methods.deleteAgentTriggerDeliveriesByUser).not.toHaveBeenCalled();
+ await service.stop();
+ });
+
+ it("drains without data loss, then purges one user's deliveries after commit", async () => {
+ const countActiveAgentTriggerDeliveriesByUser = jest
+ .fn()
+ .mockResolvedValueOnce(1)
+ .mockResolvedValueOnce(0);
+ const deleteAgentTriggerDeliveriesByUser = jest.fn(async () => {
+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
+ });
+ const methods = deliveryMethods({
+ countActiveAgentTriggerDeliveriesByUser,
+ deleteAgentTriggerDeliveriesByUser,
+ });
+ const service = createAgentTriggerService({
+ methods,
+ userDrainPollMs: 1,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+
+ await service.drainUser('507f1f77bcf86cd799439011');
+
+ expect(countActiveAgentTriggerDeliveriesByUser).toHaveBeenCalledTimes(2);
+ expect(deleteAgentTriggerDeliveriesByUser).not.toHaveBeenCalled();
+
+ await service.stop();
+ await service.purgeUser('507f1f77bcf86cd799439011');
+
+ expect(deleteAgentTriggerDeliveriesByUser).toHaveBeenCalledWith('507f1f77bcf86cd799439011');
+ });
+
+ it('arms and disarms post-commit purge recovery in system context', async () => {
+ const prepareAgentTriggerUserPurge = jest.fn(async () => {
+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
+ });
+ const cancelAgentTriggerUserPurge = jest.fn(async () => {
+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
+ return true;
+ });
+ const methods = deliveryMethods({
+ prepareAgentTriggerUserPurge,
+ cancelAgentTriggerUserPurge,
+ });
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+ const fence = new Date(START);
+
+ await service.prepareUserPurge('507f1f77bcf86cd799439011', fence, 'tenant-1');
+ await expect(service.cancelUserPurge('507f1f77bcf86cd799439011', fence)).resolves.toBe(true);
+
+ expect(prepareAgentTriggerUserPurge).toHaveBeenCalledWith(
+ '507f1f77bcf86cd799439011',
+ fence,
+ 'tenant-1',
+ );
+ await service.stop();
+ });
+
+ it('retries recoverable post-commit purges after a transient sweep failure', async () => {
+ const recoverAgentTriggerUserPurges = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('mongo unavailable'))
+ .mockResolvedValue(1);
+ const service = createAgentTriggerService({
+ methods: deliveryMethods({ recoverAgentTriggerUserPurges }),
+ purgeRecoveryIntervalMs: 5,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ expect(recoverAgentTriggerUserPurges.mock.calls.length).toBeGreaterThanOrEqual(2);
+ await service.stop();
+ });
+
+ it('keeps trusted operational reads and requeue in system tenant context', async () => {
+ const getAgentTriggerDeadLetters = jest.fn(async () => {
+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
+ return [deliveryRecord({ status: 'dead' })];
+ });
+ const requeueAgentTriggerDelivery = jest.fn(async () => {
+ expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
+ return deliveryRecord();
+ });
+ const methods = deliveryMethods({
+ getAgentTriggerDeadLetters,
+ requeueAgentTriggerDelivery,
+ });
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 60_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+
+ await expect(service.getDeadLetters(10)).resolves.toHaveLength(1);
+ await expect(service.requeue('delivery-row-1', START)).resolves.toMatchObject({
+ status: 'pending',
+ });
+ await service.stop();
+ });
+
+ it('does not become ready when the required unique indexes fail', async () => {
+ const methods = deliveryMethods({
+ ensureAgentTriggerDeliveryIndexes: jest.fn(async () =>
+ Promise.reject(new Error('index unavailable')),
+ ),
+ });
+ const service = createAgentTriggerService({ methods });
+
+ await expect(
+ service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ }),
+ ).rejects.toThrow('index unavailable');
+ await expect(service.enqueue(envelope())).rejects.toBeInstanceOf(
+ AgentTriggerServiceUnavailableError,
+ );
+ await service.stop();
+ });
+});
diff --git a/packages/api/src/agents/triggers/service.spec.ts b/packages/api/src/agents/triggers/service.spec.ts
new file mode 100644
index 0000000000..51fe993731
--- /dev/null
+++ b/packages/api/src/agents/triggers/service.spec.ts
@@ -0,0 +1,126 @@
+import jwt from 'jsonwebtoken';
+import type { AgentTriggerFetch } from './host';
+import { createAgentTriggerEnvelope } from './envelope';
+import { AGENT_TRIGGER_SCOPE } from '../../crypto/jwt';
+import { createAgentTriggerService } from './service';
+import { AgentTriggerExecutionError } from './host';
+
+const envelope = () =>
+ createAgentTriggerEnvelope({
+ mode: 'fire',
+ requestId: 'request-1',
+ deliveryId: 'delivery-1',
+ receivedAt: 20,
+ principal: { id: 'user-1', role: 'member', tenantId: 'tenant-1' },
+ target: { agentId: 'agent-1' },
+ event: {
+ id: 'event-1',
+ type: 'resource.ready',
+ occurredAt: 10,
+ source: { id: 'source-1', type: 'webhook' },
+ },
+ input: 'Handle the ready resource.',
+ });
+
+const accepted = () =>
+ new Response(
+ JSON.stringify({
+ status: 'started',
+ streamId: 'conversation-1',
+ conversationId: 'conversation-1',
+ generationCreatedAt: 25,
+ }),
+ { status: 200, headers: { 'content-type': 'application/json' } },
+ );
+
+describe('createAgentTriggerService', () => {
+ const originalSecret = process.env.JWT_SECRET;
+ const originalSelfUrl = process.env.AGENT_TRIGGERS_SELF_URL;
+
+ beforeEach(() => {
+ process.env.JWT_SECRET = 'trigger-test-secret';
+ delete process.env.AGENT_TRIGGERS_SELF_URL;
+ });
+
+ afterAll(() => {
+ process.env.JWT_SECRET = originalSecret;
+ if (originalSelfUrl == null) {
+ delete process.env.AGENT_TRIGGERS_SELF_URL;
+ } else {
+ process.env.AGENT_TRIGGERS_SELF_URL = originalSelfUrl;
+ }
+ });
+
+ it('targets the bound listener and mints a scoped user token', async () => {
+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
+ async () => accepted(),
+ );
+ const service = createAgentTriggerService({ fetch: fetcher });
+ await service.initialize({ address: { address: '::', family: 'IPv6', port: 4123 } });
+
+ await expect(service.dispatch(envelope())).resolves.toMatchObject({
+ mode: 'fire',
+ status: 'started',
+ conversationId: 'conversation-1',
+ });
+
+ const [url, init] = fetcher.mock.calls[0];
+ expect(String(url)).toBe('http://[::1]:4123/api/agents/chat/agents');
+ const headers = new Headers(init?.headers);
+ expect(headers.get('x-lc-agent-trigger')).toBe('1');
+ const authorization = headers.get('authorization');
+ expect(authorization).toMatch(/^Bearer /);
+ expect(
+ jwt.verify(authorization!.slice('Bearer '.length), process.env.JWT_SECRET!, {
+ algorithms: ['HS256'],
+ }),
+ ).toMatchObject({ id: 'user-1', scope: AGENT_TRIGGER_SCOPE });
+ });
+
+ it('honors an operator self-URL override before listener initialization', async () => {
+ process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal/base';
+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
+ async () => accepted(),
+ );
+ const service = createAgentTriggerService({ fetch: fetcher, mintToken: () => 'token' });
+
+ await service.dispatch(envelope());
+
+ expect(String(fetcher.mock.calls[0][0])).toBe(
+ 'https://triggers.internal/base/api/agents/chat/agents',
+ );
+ });
+
+ it('rechecks the durable principal before dispatching queued or direct work', async () => {
+ process.env.AGENT_TRIGGERS_SELF_URL = 'https://triggers.internal';
+ const fetcher = jest.fn(async () => accepted());
+ const service = createAgentTriggerService({
+ fetch: fetcher,
+ mintToken: () => 'token',
+ isPrincipalActive: async () => false,
+ });
+
+ await expect(service.dispatch(envelope())).rejects.toThrow(
+ 'Agent trigger delivery principal is no longer active',
+ );
+ expect(fetcher).not.toHaveBeenCalled();
+ });
+
+ it('fails safely when dispatch starts before a listener or override exists', async () => {
+ expect.hasAssertions();
+ const service = createAgentTriggerService({
+ fetch: jest.fn(async () => accepted()),
+ mintToken: () => 'token',
+ });
+
+ await service.dispatch(envelope()).catch((error: unknown) => {
+ expect(error).toBeInstanceOf(AgentTriggerExecutionError);
+ expect(error).toMatchObject({
+ mode: 'fire',
+ certainty: 'definite',
+ retryable: true,
+ code: 'SETUP_FAILED',
+ });
+ });
+ });
+});
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
new file mode 100644
index 0000000000..ad8e155821
--- /dev/null
+++ b/packages/api/src/agents/triggers/service.ts
@@ -0,0 +1,414 @@
+import { logger, runAsSystem } from '@librechat/data-schemas';
+import type {
+ AgentTriggerDeliveryFailure,
+ AgentTriggerDeliveryEngine,
+ AgentTriggerDeliveryEngineOptions,
+ AgentTriggerDeliveryRecord,
+ AgentTriggerDeliveryStore,
+} from './engine';
+import type {
+ AgentTriggerExecutionHost,
+ AgentTriggerExecutionHostDeps,
+ AgentTriggerExecutionResult,
+} from './host';
+import type { AgentTriggerEnqueueOptions, PreparedAgentTriggerDelivery } from './delivery';
+import type { BoundAddress } from '../../app/origin';
+import { AgentTriggerDeliveryDeferredError, createAgentTriggerDeliveryEngine } from './engine';
+import { isShutdownInProgress, registerShutdownTask } from '../../app/shutdown';
+import { generateAgentTriggerToken } from '../../crypto/jwt';
+import { selfOriginFromAddress } from '../../app/origin';
+import { prepareAgentTriggerDelivery } from './delivery';
+import { createAgentTriggerExecutionHost } from './host';
+import { parseAgentTriggerEnvelope } from './envelope';
+
+export const AGENT_TRIGGER_TOKEN_TTL = '60s';
+const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;
+const DEFAULT_USER_DRAIN_POLL_MS = 100;
+const DEFAULT_PURGE_RECOVERY_INTERVAL_MS = 30_000;
+const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
+
+export interface AgentTriggerServiceOptions {
+ address?: BoundAddress | string | null;
+}
+
+export interface AgentTriggerServiceDeps {
+ fetch?: AgentTriggerExecutionHostDeps['fetch'];
+ getTimezone?: AgentTriggerExecutionHostDeps['getTimezone'];
+ mintToken?: AgentTriggerExecutionHostDeps['mintToken'];
+ timeoutMs?: number;
+ methods?: AgentTriggerDeliveryPersistence;
+ deliveryOptions?: AgentTriggerDeliveryEngineOptions;
+ isPrincipalActive?: (userId: string) => boolean | Promise<boolean>;
+ userDrainTimeoutMs?: number;
+ userDrainPollMs?: number;
+ purgeRecoveryIntervalMs?: number;
+ purgeRecoveryLimit?: number;
+}
+
+export interface AgentTriggerDeliveryReceipt {
+ id: string;
+ deliveryKey: string;
+ status: AgentTriggerStoredRecord['status'];
+ availableAt: Date;
+ replayed: boolean;
+}
+
+export class AgentTriggerServiceUnavailableError extends AgentTriggerDeliveryDeferredError {
+ constructor(message: string) {
+ super(message);
+ this.name = 'AgentTriggerServiceUnavailableError';
+ }
+}
+
+export interface AgentTriggerStoredHistoryEntry {
+ attempt: number;
+ outcome: 'succeeded' | 'retry' | 'dead';
+ at: Date;
+ workerId: string;
+ error?: AgentTriggerDeliveryFailure;
+}
+
+export interface AgentTriggerStoredRecord extends Omit<AgentTriggerDeliveryRecord, 'claimToken'> {
+ claimToken?: string;
+ tenantId?: string;
+ result?: unknown;
+ history?: AgentTriggerStoredHistoryEntry[];
+ settledAt?: Date;
+ expiresAt?: Date;
+ requeueCount?: number;
+}
+
+/** Persistence contract implemented by the data-schemas method bundle. */
+export interface AgentTriggerDeliveryPersistence {
+ ensureAgentTriggerDeliveryIndexes: () => Promise<void>;
+ enqueueAgentTriggerDelivery: (
+ input: PreparedAgentTriggerDelivery,
+ ) => Promise<{ delivery: AgentTriggerStoredRecord; replayed: boolean }>;
+ claimNextAgentTriggerDelivery: AgentTriggerDeliveryStore['claimNext'];
+ findEarlierAgentTriggerDelivery: AgentTriggerDeliveryStore['findEarlierUnsettled'];
+ releaseAgentTriggerDelivery: AgentTriggerDeliveryStore['release'];
+ beginAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['beginAttempt'];
+ deferAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['defer'];
+ completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];
+ retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
+ deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];
+ getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
+ getAgentTriggerDeadLetters: (limit?: number) => Promise<AgentTriggerStoredRecord[]>;
+ requeueAgentTriggerDelivery: (
+ id: string,
+ availableAt: Date,
+ ) => Promise<AgentTriggerStoredRecord | null>;
+ countActiveAgentTriggerDeliveriesByUser: (userId: string, now: Date) => Promise<number>;
+ recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;
+ reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;
+ prepareAgentTriggerUserPurge: (
+ userId: string,
+ fenceStartedAt: Date,
+ tenantId?: string,
+ ) => Promise<void>;
+ cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
+ recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
+ deleteAgentTriggerDeliveriesByUser: (userId: string) => Promise<void>;
+}
+
+export interface AgentTriggerService {
+ initialize: (options?: AgentTriggerServiceOptions) => Promise<void>;
+ stop: () => Promise<void>;
+ dispatch: (
+ envelope: unknown,
+ options?: { signal?: AbortSignal },
+ ) => Promise<AgentTriggerExecutionResult>;
+ enqueue: (
+ envelope: unknown,
+ options?: AgentTriggerEnqueueOptions,
+ ) => Promise<AgentTriggerDeliveryReceipt>;
+ getDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
+ getDeadLetters: (limit?: number) => Promise<AgentTriggerStoredRecord[]>;
+ requeue: (id: string, availableAt?: Date) => Promise<AgentTriggerStoredRecord | null>;
+ drainUser: (userId: string) => Promise<void>;
+ prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
+ cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
+ purgeUser: (userId: string) => Promise<void>;
+}
+
+function createDeliveryStore(methods: AgentTriggerDeliveryPersistence): AgentTriggerDeliveryStore {
+ return {
+ claimNext: methods.claimNextAgentTriggerDelivery,
+ findEarlierUnsettled: methods.findEarlierAgentTriggerDelivery,
+ release: methods.releaseAgentTriggerDelivery,
+ beginAttempt: methods.beginAgentTriggerDeliveryAttempt,
+ defer: methods.deferAgentTriggerDeliveryAttempt,
+ complete: methods.completeAgentTriggerDelivery,
+ retry: methods.retryAgentTriggerDelivery,
+ dead: methods.deadLetterAgentTriggerDelivery,
+ };
+}
+
+function requireDeliveryOrigin(boundOrigin: string | undefined): void {
+ const value = process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin;
+ let url: URL;
+ try {
+ url = new URL(value ?? '');
+ } catch {
+ throw new AgentTriggerServiceUnavailableError(
+ 'Durable agent trigger delivery requires a valid listener address or AGENT_TRIGGERS_SELF_URL',
+ );
+ }
+ if ((url.protocol !== 'http:' && url.protocol !== 'https:') || url.username || url.password) {
+ throw new AgentTriggerServiceUnavailableError(
+ 'Durable agent trigger delivery requires an HTTP(S) self URL without credentials',
+ );
+ }
+}
+
+/** Production composition for trusted, in-process trigger producers. */
+export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): AgentTriggerService {
+ const userDrainTimeoutMs = deps.userDrainTimeoutMs ?? DEFAULT_USER_DRAIN_TIMEOUT_MS;
+ const userDrainPollMs = deps.userDrainPollMs ?? DEFAULT_USER_DRAIN_POLL_MS;
+ const purgeRecoveryIntervalMs =
+ deps.purgeRecoveryIntervalMs ?? DEFAULT_PURGE_RECOVERY_INTERVAL_MS;
+ const purgeRecoveryLimit = deps.purgeRecoveryLimit ?? DEFAULT_PURGE_RECOVERY_LIMIT;
+ if (!Number.isSafeInteger(userDrainTimeoutMs) || userDrainTimeoutMs <= 0) {
+ throw new TypeError('userDrainTimeoutMs must be a positive integer');
+ }
+ if (!Number.isSafeInteger(userDrainPollMs) || userDrainPollMs <= 0) {
+ throw new TypeError('userDrainPollMs must be a positive integer');
+ }
+ if (!Number.isSafeInteger(purgeRecoveryIntervalMs) || purgeRecoveryIntervalMs <= 0) {
+ throw new TypeError('purgeRecoveryIntervalMs must be a positive integer');
+ }
+ if (!Number.isSafeInteger(purgeRecoveryLimit) || purgeRecoveryLimit <= 0) {
+ throw new TypeError('purgeRecoveryLimit must be a positive integer');
+ }
+ let boundOrigin: string | undefined;
+ let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
+ let initializePromise: Promise<void> | undefined;
+ let purgeRecoveryPromise: Promise<void> | undefined;
+ let purgeRecoveryTimer: NodeJS.Timeout | undefined;
+ let deliveryReady = false;
+ let stopping = false;
+ const isPrincipalActive = deps.isPrincipalActive;
+ const host: AgentTriggerExecutionHost = createAgentTriggerExecutionHost({
+ getBaseUrl: () => {
+ const origin = process.env.AGENT_TRIGGERS_SELF_URL ?? boundOrigin;
+ if (origin == null) {
+ throw new Error('Agent trigger service has not been initialized with a listener address');
+ }
+ return origin;
+ },
+ mintToken:
+ deps.mintToken ??
+ ((principal) => generateAgentTriggerToken(principal.userId, AGENT_TRIGGER_TOKEN_TTL)),
+ ...(deps.fetch != null && { fetch: deps.fetch }),
+ ...(deps.getTimezone != null && { getTimezone: deps.getTimezone }),
+ ...(deps.timeoutMs != null && { timeoutMs: deps.timeoutMs }),
+ });
+
+ const requireMethods = (): AgentTriggerDeliveryPersistence => {
+ if (deps.methods == null || !deliveryReady || isShutdownInProgress()) {
+ throw new AgentTriggerServiceUnavailableError(
+ 'Durable agent trigger delivery is not ready on this server',
+ );
+ }
+ return deps.methods;
+ };
+
+ const requireCleanupMethods = (): AgentTriggerDeliveryPersistence => {
+ if (deps.methods == null) {
+ throw new AgentTriggerServiceUnavailableError(
+ 'Durable agent trigger delivery is not configured on this server',
+ );
+ }
+ return deps.methods;
+ };
+
+ const requireActivePrincipal = async (userId: string): Promise<void> => {
+ if (isPrincipalActive != null && !(await runAsSystem(async () => isPrincipalActive(userId)))) {
+ throw new AgentTriggerServiceUnavailableError(
+ 'Agent trigger delivery principal is no longer active',
+ );
+ }
+ };
+
+ const dispatchForActivePrincipal = async (
+ envelope: unknown,
+ options?: { signal?: AbortSignal },
+ ): Promise<AgentTriggerExecutionResult> => {
+ const parsed = parseAgentTriggerEnvelope(envelope);
+ await requireActivePrincipal(parsed.principal.userId);
+ return host.dispatch(parsed, options);
+ };
+
+ const drainUser = async (userId: string): Promise<void> => {
+ const methods = requireMethods();
+ await deliveryEngine?.cancelUser(userId);
+
+ try {
+ const deadline = Date.now() + userDrainTimeoutMs;
+ while (
+ (await runAsSystem(async () =>
+ methods.countActiveAgentTriggerDeliveriesByUser(userId, new Date()),
+ )) > 0
+ ) {
+ if (Date.now() >= deadline) {
+ throw new AgentTriggerServiceUnavailableError(
+ `Timed out draining active agent trigger deliveries for user ${userId}`,
+ );
+ }
+ await new Promise((resolve) => setTimeout(resolve, userDrainPollMs));
+ }
+ } finally {
+ deliveryEngine?.releaseUserCancellation(userId);
+ }
+ };
+
+ const recoverPurges = (): Promise<void> => {
+ if (deps.methods == null || stopping) {
+ return Promise.resolve();
+ }
+ if (purgeRecoveryPromise != null) {
+ return purgeRecoveryPromise;
+ }
+ const methods = deps.methods;
+ const current = runAsSystem(async () => {
+ const [purgedUsers, publishedLanes, reclaimedLanes] = await Promise.all([
+ methods.recoverAgentTriggerUserPurges(purgeRecoveryLimit),
+ methods.recoverAgentTriggerLanePublications(purgeRecoveryLimit),
+ methods.reclaimInactiveAgentTriggerLanes(purgeRecoveryLimit),
+ ]);
+ if (publishedLanes > 0) {
+ deliveryEngine?.wake();
+ }
+ if (purgedUsers > 0 || publishedLanes > 0 || reclaimedLanes > 0) {
+ logger.info('[agent-triggers] recovered durable delivery maintenance', {
+ purgedUsers,
+ publishedLanes,
+ reclaimedLanes,
+ });
+ }
+ })
+ .catch((error) => {
+ logger.error('[agent-triggers] durable delivery maintenance failed:', error);
+ })
+ .finally(() => {
+ if (purgeRecoveryPromise === current) {
+ purgeRecoveryPromise = undefined;
+ }
+ });
+ purgeRecoveryPromise = current;
+ return current;
+ };
+
+ const startPurgeRecovery = (): void => {
+ if (purgeRecoveryTimer != null) {
+ return;
+ }
+ void recoverPurges();
+ purgeRecoveryTimer = setInterval(() => void recoverPurges(), purgeRecoveryIntervalMs);
+ purgeRecoveryTimer.unref();
+ };
+
+ const stop = async (): Promise<void> => {
+ stopping = true;
+ deliveryReady = false;
+ if (purgeRecoveryTimer != null) {
+ clearInterval(purgeRecoveryTimer);
+ purgeRecoveryTimer = undefined;
+ }
+ await initializePromise?.catch(() => undefined);
+ await deliveryEngine?.stop();
+ await purgeRecoveryPromise?.catch(() => undefined);
+ };
+
+ if (deps.methods != null) {
+ registerShutdownTask('agent trigger delivery engine', stop, {
+ phase: 'pre-drain',
+ priority: 100,
+ });
+ }
+
+ return {
+ initialize: (options = {}) => {
+ boundOrigin = selfOriginFromAddress(options.address) ?? boundOrigin;
+ if (deps.methods == null || deliveryReady) {
+ return Promise.resolve();
+ }
+ if (initializePromise != null) {
+ return initializePromise;
+ }
+ if (stopping || isShutdownInProgress()) {
+ return Promise.reject(
+ new AgentTriggerServiceUnavailableError(
+ 'Durable agent trigger delivery cannot start during shutdown',
+ ),
+ );
+ }
+ const methods = deps.methods;
+ initializePromise = runAsSystem(async () => {
+ requireDeliveryOrigin(boundOrigin);
+ await methods.ensureAgentTriggerDeliveryIndexes();
+ if (stopping || isShutdownInProgress()) {
+ throw new AgentTriggerServiceUnavailableError(
+ 'Durable agent trigger delivery cannot start during shutdown',
+ );
+ }
+ deliveryEngine = createAgentTriggerDeliveryEngine(
+ {
+ store: createDeliveryStore(methods),
+ dispatch: dispatchForActivePrincipal,
+ },
+ deps.deliveryOptions,
+ );
+ deliveryReady = true;
+ deliveryEngine.start();
+ startPurgeRecovery();
+ logger.info('[agent-triggers] durable delivery engine started');
+ }).finally(() => {
+ initializePromise = undefined;
+ });
+ return initializePromise;
+ },
+ stop,
+ dispatch: dispatchForActivePrincipal,
+ enqueue: async (envelope, options) => {
+ const methods = requireMethods();
+ const prepared = prepareAgentTriggerDelivery(envelope, options);
+ await requireActivePrincipal(String(prepared.user));
+ const queued = await runAsSystem(async () => methods.enqueueAgentTriggerDelivery(prepared));
+ try {
+ await requireActivePrincipal(String(prepared.user));
+ } catch (error) {
+ await drainUser(String(prepared.user));
+ throw error;
+ }
+ deliveryEngine?.wake();
+ return {
+ id: queued.delivery.id,
+ deliveryKey: queued.delivery.deliveryKey,
+ status: queued.delivery.status,
+ availableAt: queued.delivery.availableAt,
+ replayed: queued.replayed,
+ };
+ },
+ getDelivery: (deliveryKey) =>
+ runAsSystem(async () => requireMethods().getAgentTriggerDelivery(deliveryKey)),
+ getDeadLetters: (limit) =>
+ runAsSystem(async () => requireMethods().getAgentTriggerDeadLetters(limit)),
+ requeue: (id, availableAt = new Date()) =>
+ runAsSystem(async () => requireMethods().requeueAgentTriggerDelivery(id, availableAt)),
+ drainUser,
+ prepareUserPurge: (userId, fenceStartedAt, tenantId) =>
+ runAsSystem(async () =>
+ requireCleanupMethods().prepareAgentTriggerUserPurge(userId, fenceStartedAt, tenantId),
+ ),
+ cancelUserPurge: (userId, fenceStartedAt) =>
+ runAsSystem(async () =>
+ requireCleanupMethods().cancelAgentTriggerUserPurge(userId, fenceStartedAt),
+ ),
+ // Account deletion may reach this post-commit cleanup after graceful
+ // shutdown has begun. Persistence remains usable even though admissions
+ // and the delivery engine are deliberately no longer ready.
+ purgeUser: (userId) =>
+ runAsSystem(async () => requireCleanupMethods().deleteAgentTriggerDeliveriesByUser(userId)),
+ };
+}
diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.ts
index b5962a45f7..8dff0f8f9f 100644
--- a/packages/api/src/app/index.ts
+++ b/packages/api/src/app/index.ts
@@ -7,5 +7,6 @@ export * from './checks';
export * from './resolve';
export * from './shutdown';
export * from './server';
+export * from './origin';
export { resolveBuildInfo } from './build';
export type { BuildInfo } from './build';
diff --git a/packages/api/src/app/origin.ts b/packages/api/src/app/origin.ts
new file mode 100644
index 0000000000..883111d38d
--- /dev/null
+++ b/packages/api/src/app/origin.ts
@@ -0,0 +1,21 @@
+/** A listener's `server.address()`. */
+export interface BoundAddress {
+ address: string;
+ family?: string | number;
+ port: number;
+}
+
+/** Resolve an HTTP origin that reaches the current process's bound listener. */
+export function selfOriginFromAddress(
+ address: BoundAddress | string | null | undefined,
+): string | null {
+ if (address == null || typeof address === 'string' || !address.port) {
+ return null;
+ }
+ const bound = address.address;
+ const isIPv6 = address.family === 'IPv6' || address.family === 6 || bound.includes(':');
+ const loopback = isIPv6 ? '::1' : '127.0.0.1';
+ const isWildcard = bound === '' || bound === '::' || bound === '0.0.0.0';
+ const host = isWildcard ? loopback : bound;
+ return `http://${isIPv6 ? `[${host}]` : host}:${address.port}`;
+}
diff --git a/packages/api/src/app/shutdown.spec.ts b/packages/api/src/app/shutdown.spec.ts
index 7733bb5fed..16f2de5c51 100644
--- a/packages/api/src/app/shutdown.spec.ts
+++ b/packages/api/src/app/shutdown.spec.ts
@@ -2,6 +2,7 @@
import http from 'http';
import {
setupGracefulShutdown,
+ isShutdownInProgress,
registerShutdownTask,
__resetShutdownStateForTests,
} from './shutdown';
@@ -160,6 +161,26 @@ describe('setupGracefulShutdown', () => {
expect(closeSpy).toHaveBeenCalledTimes(1);
});
+ it('exposes shutdown state before pre-drain work begins', async () => {
+ let observedDuringPreDrain = false;
+ jest.spyOn(server, 'close').mockImplementation(() => server);
+ registerShutdownTask(
+ 'observe-shutdown',
+ () => {
+ observedDuringPreDrain = isShutdownInProgress();
+ },
+ { phase: 'pre-drain' },
+ );
+ setupGracefulShutdown(server);
+
+ expect(isShutdownInProgress()).toBe(false);
+ triggerSignal('SIGTERM');
+ await flush();
+
+ expect(observedDuringPreDrain).toBe(true);
+ expect(isShutdownInProgress()).toBe(true);
+ });
+
it('force-exits with code 1 if shutdown exceeds the timeout', () => {
jest.useFakeTimers();
jest.spyOn(server, 'close').mockImplementation(() => server);
diff --git a/packages/api/src/app/shutdown.ts b/packages/api/src/app/shutdown.ts
index dd131fb7d9..e8662296d5 100644
--- a/packages/api/src/app/shutdown.ts
+++ b/packages/api/src/app/shutdown.ts
@@ -42,6 +42,11 @@ export function registerShutdownTask(
tasks.push({ name, fn, phase, priority, registrationOrder: nextRegistrationOrder++ });
}
+/** Whether graceful shutdown has started, for admission paths that must fail closed. */
+export function isShutdownInProgress(): boolean {
+ return isShuttingDown;
+}
+
/**
* Wires SIGTERM, SIGINT, SIGQUIT, and SIGHUP to a graceful shutdown
* sequence: initiate HTTP server close to stop accepting new connections,
diff --git a/packages/api/src/crypto/jwt.spec.ts b/packages/api/src/crypto/jwt.spec.ts
new file mode 100644
index 0000000000..ccf2d04ef8
--- /dev/null
+++ b/packages/api/src/crypto/jwt.spec.ts
@@ -0,0 +1,52 @@
+import {
+ AGENT_TRIGGER_SCOPE,
+ generateAgentTriggerToken,
+ generateShortLivedToken,
+ isAgentTriggerRequest,
+} from './jwt';
+
+function request(token: string, marker = true) {
+ return {
+ headers: {
+ ...(marker && { 'x-lc-agent-trigger': '1' }),
+ authorization: `Bearer ${token}`,
+ },
+ };
+}
+
+describe('agent trigger identity', () => {
+ const original = process.env.JWT_SECRET;
+
+ beforeAll(() => {
+ process.env.JWT_SECRET = 'test-secret';
+ });
+
+ afterAll(() => {
+ if (original == null) {
+ delete process.env.JWT_SECRET;
+ } else {
+ process.env.JWT_SECRET = original;
+ }
+ });
+
+ it('recognizes only a signed trigger scope with its transport marker', () => {
+ const trigger = generateAgentTriggerToken('user-1');
+ const ordinary = generateShortLivedToken('user-1');
+
+ expect(isAgentTriggerRequest(request(trigger))).toBe(true);
+ expect(isAgentTriggerRequest(request(trigger, false))).toBe(false);
+ expect(isAgentTriggerRequest(request(ordinary))).toBe(false);
+ expect(isAgentTriggerRequest(request('invalid'))).toBe(false);
+ });
+
+ it('uses the dedicated trigger scope without changing ordinary tokens', () => {
+ const trigger = generateAgentTriggerToken('user-1');
+ const payload = JSON.parse(Buffer.from(trigger.split('.')[1], 'base64url').toString()) as {
+ id: string;
+ scope: string;
+ };
+
+ expect(payload).toMatchObject({ id: 'user-1', scope: AGENT_TRIGGER_SCOPE });
+ expect(generateShortLivedToken('user-1')).not.toBe(trigger);
+ });
+});
diff --git a/packages/api/src/crypto/jwt.ts b/packages/api/src/crypto/jwt.ts
index 380fc613a1..db6d8140d3 100644
--- a/packages/api/src/crypto/jwt.ts
+++ b/packages/api/src/crypto/jwt.ts
@@ -1,10 +1,17 @@
import jwt from 'jsonwebtoken';
+export const AGENT_TRIGGER_SCOPE = 'agent_trigger';
+
+type AgentTriggerRequest = {
+ headers?: Record<string, string | string[] | undefined>;
+ _isAgentTrigger?: boolean;
+};
+
/**
- * Generate a short-lived JWT token
- * @param {String} userId - The ID of the user
- * @param {String} [expireIn='5m'] - The expiration time for the token (default is 5 minutes)
- * @returns {String} - The generated JWT token
+ * Generate a short-lived JWT token.
+ * @param {String} userId - The ID of the user.
+ * @param {String} [expireIn='5m'] - The expiration time for the token.
+ * @returns {String} - The generated JWT token.
*/
export const generateShortLivedToken = (userId: string, expireIn: string = '5m'): string => {
return jwt.sign({ id: userId }, process.env.JWT_SECRET!, {
@@ -12,3 +19,33 @@ export const generateShortLivedToken = (userId: string, expireIn: string = '5m')
algorithm: 'HS256',
});
};
+
+/** Mint the server-only identity used by durable agent trigger admission. */
+export const generateAgentTriggerToken = (userId: string, expireIn: string = '60s'): string => {
+ return jwt.sign({ id: userId, scope: AGENT_TRIGGER_SCOPE }, process.env.JWT_SECRET!, {
+ expiresIn: expireIn,
+ algorithm: 'HS256',
+ });
+};
+
+/** Verify the signed trigger scope together with its explicit transport marker. */
+export const isAgentTriggerRequest = (req?: AgentTriggerRequest): boolean => {
+ if (req?.headers?.['x-lc-agent-trigger'] !== '1') {
+ return false;
+ }
+ const auth = req.headers.authorization;
+ const token = typeof auth === 'string' && auth.startsWith('Bearer ') ? auth.slice(7) : undefined;
+ if (token == null) {
+ return false;
+ }
+ try {
+ const payload = jwt.verify(token, process.env.JWT_SECRET!, { algorithms: ['HS256'] });
+ return typeof payload === 'object' && payload.scope === AGENT_TRIGGER_SCOPE;
+ } catch {
+ return false;
+ }
+};
+
+/** Skip the shared loopback IP bucket while retaining per-user and concurrency limits. */
+export const exemptAgentTriggerFromIpLimiter = (req?: AgentTriggerRequest): boolean =>
+ typeof req?._isAgentTrigger === 'boolean' ? req._isAgentTrigger : isAgentTriggerRequest(req);
diff --git a/packages/api/src/crypto/limiters.spec.ts b/packages/api/src/crypto/limiters.spec.ts
new file mode 100644
index 0000000000..2aa6f807f2
--- /dev/null
+++ b/packages/api/src/crypto/limiters.spec.ts
@@ -0,0 +1,50 @@
+import {
+ exemptAgentTriggerFromIpLimiter,
+ generateAgentTriggerToken,
+ generateShortLivedToken,
+ isAgentTriggerRequest,
+} from './jwt';
+
+describe('agent trigger IP limiter identity', () => {
+ const original = process.env.JWT_SECRET;
+
+ beforeAll(() => {
+ process.env.JWT_SECRET = 'test-secret';
+ });
+
+ afterAll(() => {
+ if (original == null) {
+ delete process.env.JWT_SECRET;
+ } else {
+ process.env.JWT_SECRET = original;
+ }
+ });
+
+ it('exempts only a signed trigger from the shared loopback IP bucket', () => {
+ const token = generateAgentTriggerToken('user-1');
+ const req = {
+ headers: {
+ 'x-lc-agent-trigger': '1',
+ authorization: `Bearer ${token}`,
+ },
+ };
+
+ expect(exemptAgentTriggerFromIpLimiter(req)).toBe(true);
+ expect(exemptAgentTriggerFromIpLimiter({ headers: {} })).toBe(false);
+ expect(
+ exemptAgentTriggerFromIpLimiter({
+ headers: {
+ 'x-lc-agent-trigger': '1',
+ authorization: `Bearer ${generateShortLivedToken('user-1')}`,
+ },
+ }),
+ ).toBe(false);
+ });
+
+ it('retains the router-captured decision after the short-lived token expires', () => {
+ const req = { headers: { authorization: 'Bearer expired' }, _isAgentTrigger: true };
+
+ expect(isAgentTriggerRequest(req)).toBe(false);
+ expect(exemptAgentTriggerFromIpLimiter(req)).toBe(true);
+ });
+});
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 9014d12602..5078051aa7 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -91,6 +91,8 @@ const REAPED_JOB_ERROR = 'Generation timed out';
* digits; this trips only when Redis stalls, bounding buffered batches and
* queued commands by pacing the producer instead of growing without limit. */
const MAX_OUTSTANDING_COALESCED_RECEIPTS = 256;
+const PROVIDER_DRAIN_TIMEOUT_MS = 30_000;
+const PROVIDER_DRAIN_POLL_MS = 50;
/** Bounded completed-request replay horizon. It exceeds the default 24-hour
* approval window; if a custom/live job outlasts it, `resumeClaimedGeneration`
@@ -1563,16 +1565,37 @@ class GenerationJobManagerClass {
// resumed owner must acknowledge like any other live provider.
const hadRunningProvider =
receipt.status === 'running' && receipt.providerAbortReady !== false;
- if (
- await this.notifyReplacedGeneration(
- streamId,
- receipt.createdAt,
- receipt.conversationId,
- fallbackConversationId,
- job.creationAttemptId,
- hadRunningProvider,
- )
- ) {
+ let delivered = await this.notifyReplacedGeneration(
+ streamId,
+ receipt.createdAt,
+ receipt.conversationId,
+ fallbackConversationId,
+ job.creationAttemptId,
+ hadRunningProvider,
+ );
+ if (delivered && receipt.providerDrained === false) {
+ if (!receipt.providerExecutionId) {
+ logger.error(
+ `[GenerationJobManager] Replacement receipt lacks provider identity: ${streamId}/${receipt.createdAt}`,
+ );
+ delivered = false;
+ } else {
+ try {
+ await this.waitForProviderExecutionDrain(
+ streamId,
+ receipt.createdAt,
+ receipt.providerExecutionId,
+ );
+ } catch (error) {
+ logger.error(
+ `[GenerationJobManager] Replaced provider did not drain: ${streamId}/${receipt.createdAt}`,
+ error,
+ );
+ delivered = false;
+ }
+ }
+ }
+ if (delivered) {
acknowledged.push(receipt.createdAt);
} else {
allReceiptsDelivered = false;
@@ -1844,61 +1867,76 @@ class GenerationJobManagerClass {
* terminal hash that makes every duplicate retry wait until Redis TTL expiry. */
private async terminalizeUnexposedGeneration(
streamId: string,
- job: Pick<SerializableJobData, 'createdAt' | 'conversationId'>,
+ job: Pick<SerializableJobData, 'createdAt' | 'conversationId' | 'providerExecutionId'>,
message: string,
): Promise<boolean> {
- const finalEvent: t.FinalEvent = {
- final: true,
- reconcile: true,
- reconcileReason: 'terminal_payload_missing',
- terminalStatus: 'error',
- generationCreatedAt: job.createdAt,
- conversation: { conversationId: job.conversationId ?? streamId },
- };
- let lastError: unknown;
- for (let attempt = 0; attempt < 2; attempt++) {
- try {
- if (
- await this.jobStore.transitionStatus(streamId, {
- from: 'running',
- to: 'error',
- patch: {
- completedAt: Date.now(),
- error: message,
- finalEvent: JSON.stringify(finalEvent),
- },
- expectCreatedAt: job.createdAt,
- })
- ) {
- return true;
+ try {
+ const finalEvent: t.FinalEvent = {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'error',
+ generationCreatedAt: job.createdAt,
+ conversation: { conversationId: job.conversationId ?? streamId },
+ };
+ let lastError: unknown;
+ for (let attempt = 0; attempt < 2; attempt++) {
+ try {
+ if (
+ await this.jobStore.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ patch: {
+ completedAt: Date.now(),
+ error: message,
+ finalEvent: JSON.stringify(finalEvent),
+ },
+ expectCreatedAt: job.createdAt,
+ })
+ ) {
+ return true;
+ }
+ } catch (error) {
+ // As with create, the terminal CAS may have committed before the reply
+ // was lost. Probe its exact epoch before retrying.
+ lastError = error;
}
- } catch (error) {
- // As with create, the terminal CAS may have committed before the reply
- // was lost. Probe its exact epoch before retrying.
- lastError = error;
- }
- try {
- const current = await this.jobStore.getJob(streamId);
- if (
- current == null ||
- current.createdAt !== job.createdAt ||
- (current.status !== 'running' && current.status !== 'requires_action')
- ) {
- return true;
+ try {
+ const current = await this.jobStore.getJob(streamId);
+ if (
+ current == null ||
+ current.createdAt !== job.createdAt ||
+ (current.status !== 'running' && current.status !== 'requires_action')
+ ) {
+ return true;
+ }
+ } catch (error) {
+ lastError = error;
}
- } catch (error) {
- lastError = error;
}
- }
- if (lastError != null) {
- logger.error(
- `[GenerationJobManager] Failed to terminalize unexposed generation ${streamId}:`,
- lastError,
- );
+ if (lastError != null) {
+ logger.error(
+ `[GenerationJobManager] Failed to terminalize unexposed generation ${streamId}:`,
+ lastError,
+ );
+ }
+ return false;
+ } finally {
+ if (job.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.providerExecutionId,
+ ).catch((error) => {
+ logger.error(
+ `[GenerationJobManager] Failed to mark unexposed provider drained ${streamId}:`,
+ error,
+ );
+ });
+ }
}
- return false;
}
async createJob(
@@ -1938,7 +1976,12 @@ class GenerationJobManagerClass {
const tenantId = getTenantId();
const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;
- const initialMetadata = sanitizeJobMetadata(options.initialMetadata ?? {});
+ const creationAttemptId = randomUUID();
+ const initialMetadata = {
+ ...sanitizeJobMetadata(options.initialMetadata ?? {}),
+ providerExecutionId: creationAttemptId,
+ providerDrained: true,
+ };
if (
(options.recoveredSteerId != null) !== (options.recoveredSteerPayload != null) ||
(options.recoveredSteerPayload != null &&
@@ -1946,7 +1989,6 @@ class GenerationJobManagerClass {
) {
throw new RecoveredSteerPayloadMismatchError();
}
- const creationAttemptId = randomUUID();
// Capture the active epoch before the store atomically replaces it. A
// subscriber attached to that predecessor filters events by generation,
// so replacement must explicitly close/handoff that old attachment.
@@ -1974,6 +2016,18 @@ class GenerationJobManagerClass {
throw error;
}
if (error instanceof JobCreationSupersededError) {
+ if (error.createdJob.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ error.createdJob.createdAt,
+ error.createdJob.providerExecutionId,
+ ).catch((drainError) => {
+ logger.error(
+ `[GenerationJobManager] Failed to mark superseded unexposed provider drained ${streamId}:`,
+ drainError,
+ );
+ });
+ }
try {
const current = (await this.jobStore.getJob(streamId)) as CreatedJobData | null;
if (
@@ -2092,6 +2146,13 @@ class GenerationJobManagerClass {
currentRuntimeBeforeInstall.createdAt > jobData.createdAt) ||
(ownedCreatedAtBeforeInstall != null && ownedCreatedAtBeforeInstall > jobData.createdAt)
) {
+ if (jobData.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ jobData.createdAt,
+ jobData.providerExecutionId,
+ );
+ }
throw new Error('Generation job was replaced during initialization');
}
if (this.shuttingDown) {
@@ -2110,6 +2171,13 @@ class GenerationJobManagerClass {
patch: { completedAt: Date.now(), error: SHUTDOWN_JOB_ERROR },
expectCreatedAt: jobData.createdAt,
});
+ if (jobData.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ jobData.createdAt,
+ jobData.providerExecutionId,
+ );
+ }
const replacedLocal = this.runtimeState.get(streamId);
if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {
replacedLocal.abortController.abort();
@@ -2186,6 +2254,13 @@ class GenerationJobManagerClass {
runtimeImmediatelyBeforeInstall.createdAt > jobData.createdAt) ||
(ownedImmediatelyBeforeInstall != null && ownedImmediatelyBeforeInstall > jobData.createdAt)
) {
+ if (jobData.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ jobData.createdAt,
+ jobData.providerExecutionId,
+ );
+ }
throw new Error('Generation job was replaced during initialization');
}
@@ -2301,6 +2376,18 @@ class GenerationJobManagerClass {
finalizeError,
);
});
+ if (jobData.providerExecutionId) {
+ await this.markProviderExecutionDrained(
+ streamId,
+ jobData.createdAt,
+ jobData.providerExecutionId,
+ ).catch((drainError) => {
+ logger.error(
+ `[GenerationJobManager] Failed to mark partially initialized provider drained ${streamId}:`,
+ drainError,
+ );
+ });
+ }
if (
this.runtimeState.get(streamId) === runtime &&
runtime.replacementTransportHold !== true
@@ -2409,6 +2496,8 @@ class GenerationJobManagerClass {
// Surface the owning replica's seal capability so the steer route can
// honour it instead of probing its own (possibly older) SDK.
preemptCapable: jobData.preemptCapable,
+ providerExecutionId: jobData.providerExecutionId,
+ providerDrained: jobData.providerDrained,
steersClosed: jobData.steersClosed,
idempotencyClientRequestId: jobData.idempotencyClientRequestId,
terminalPersistencePending: jobData.terminalPersistencePending,
@@ -3403,6 +3492,7 @@ class GenerationJobManagerClass {
const terminalJob = await this.jobStore.getJob(streamId);
if (
!retainForTerminalReplay &&
+ terminalJob?.providerDrained !== false &&
(terminalJob?.createdAt !== createdAt || terminalJob.terminalPersistencePending !== true)
) {
// A same-stream replacement created after the claim makes this a safe
@@ -3538,6 +3628,9 @@ class GenerationJobManagerClass {
* same-stream replacement between authorization and the manager read
* could be stopped by a stale request intended for its predecessor. */
expectedCreatedAt?: number;
+ /** Destructive user cleanup must wait until the provider owner has
+ * completed every trailing persistence task, not merely received Stop. */
+ awaitProviderDrain?: boolean;
},
): Promise<AbortResult> {
const observedRuntime = this.runtimeState.get(streamId);
@@ -3607,6 +3700,9 @@ class GenerationJobManagerClass {
const abortableStatus = jobData.status;
if (abortableStatus !== 'running' && abortableStatus !== 'requires_action') {
+ if (options?.awaitProviderDrain) {
+ await this.waitForProviderDrainIfRequired(streamId, jobData);
+ }
this.reconcileInactiveGeneration(streamId, jobData.createdAt, jobData, observedRuntime);
logger.debug(
`[GenerationJobManager] Cannot abort terminal job ${streamId}: ${jobData.status}`,
@@ -3706,6 +3802,12 @@ class GenerationJobManagerClass {
const jobStillActive =
currentJob?.createdAt === jobData.createdAt &&
(currentJob.status === 'running' || currentJob.status === 'requires_action');
+ if (options?.awaitProviderDrain) {
+ if (jobStillActive || currentJob?.createdAt !== jobData.createdAt) {
+ throw new Error(`Failed to stop provider execution before user cleanup: ${streamId}`);
+ }
+ await this.waitForProviderDrainIfRequired(streamId, currentJob ?? jobData);
+ }
return {
success: false,
...(jobStillActive
@@ -3864,7 +3966,13 @@ class GenerationJobManagerClass {
...(publication.persistenceFailed && { persistenceFailed: true }),
};
} finally {
- await this.finishTerminalJob(terminalClaim);
+ try {
+ if (options?.awaitProviderDrain) {
+ await this.waitForProviderDrainIfRequired(streamId, jobData);
+ }
+ } finally {
+ await this.finishTerminalJob(terminalClaim);
+ }
}
}
@@ -6212,6 +6320,107 @@ class GenerationJobManagerClass {
await this.jobStore.updateJob(streamId, sanitizeJobMetadata(metadata), generationId);
}
+ /** Records that one exact provider segment has completed every trailing write.
+ * The opaque segment id prevents a paused controller from acknowledging a
+ * later HITL resume that reuses the same generation epoch. */
+ async markProviderExecutionDrained(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ if (providerExecutionId.length === 0 || providerExecutionId.length > 128) {
+ return false;
+ }
+
+ const [marked, recorded] = await Promise.all([
+ this.jobStore.markProviderExecutionDrained?.(
+ streamId,
+ expectedCreatedAt,
+ providerExecutionId,
+ ) ?? Promise.resolve(false),
+ this.eventTransport.recordProviderDrain?.(streamId, expectedCreatedAt, providerExecutionId) ??
+ Promise.resolve(false),
+ ]);
+ const job = await this.jobStore.getJob(streamId);
+ if (
+ marked &&
+ recorded &&
+ this._cleanupOnComplete &&
+ job?.createdAt === expectedCreatedAt &&
+ job.providerExecutionId === providerExecutionId &&
+ job.providerDrained === true &&
+ job.terminalPersistencePending !== true &&
+ (job.status === 'complete' || job.status === 'aborted')
+ ) {
+ await this.jobStore.deleteJob(streamId, expectedCreatedAt);
+ }
+ return marked;
+ }
+
+ /** Starts the initial provider owner only if account deletion, abort, or a
+ * replacement has not already changed the exact running generation. */
+ async beginProviderExecution(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ if (providerExecutionId.length === 0 || providerExecutionId.length > 128) {
+ return false;
+ }
+ return (
+ this.jobStore.beginProviderExecution?.(streamId, expectedCreatedAt, providerExecutionId) ??
+ Promise.resolve(false)
+ );
+ }
+
+ private async waitForProviderExecutionDrain(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<void> {
+ const deadline = Date.now() + PROVIDER_DRAIN_TIMEOUT_MS;
+ for (;;) {
+ if (
+ (await this.eventTransport
+ .hasProviderDrain?.(streamId, expectedCreatedAt, providerExecutionId)
+ .catch(() => false)) === true
+ ) {
+ return;
+ }
+ const job = await this.jobStore.getJob(streamId);
+ if (
+ job?.createdAt === expectedCreatedAt &&
+ job.providerExecutionId === providerExecutionId &&
+ job.providerDrained === true
+ ) {
+ return;
+ }
+ if (Date.now() >= deadline) {
+ throw new Error(`Timed out waiting for provider execution to drain: ${streamId}`);
+ }
+ await new Promise<void>((resolve) => {
+ setTimeout(resolve, PROVIDER_DRAIN_POLL_MS);
+ });
+ }
+ }
+
+ private async waitForProviderDrainIfRequired(
+ streamId: string,
+ job: SerializableJobData,
+ ): Promise<void> {
+ if (!job.providerExecutionId) {
+ if (job.providerDrained !== false) {
+ return;
+ }
+ throw new Error(`Cannot confirm provider execution drain for legacy job: ${streamId}`);
+ }
+ // Never trust the caller's pre-abort `providerDrained` snapshot here. The
+ // provider can win its exact begin CAS after that read but before the
+ // terminal CAS. Re-read the exact segment only after terminal ownership is
+ // committed, when no not-yet-started provider can begin anymore.
+ await this.waitForProviderExecutionDrain(streamId, job.createdAt, job.providerExecutionId);
+ }
+
/**
* Set reference to the graph's contentParts array.
*/
@@ -7122,6 +7331,12 @@ class GenerationJobManagerClass {
return this.jobStore.getActiveJobIdsByUser(userId, tenantId);
}
+ /** Returns every generation whose provider can still mutate user-owned data,
+ * including a terminal generation whose controller is finishing trailing writes. */
+ async getCleanupBlockingJobIdsForUser(userId: string, tenantId?: string): Promise<string[]> {
+ return this.jobStore.getCleanupBlockingJobIdsByUser(userId, tenantId);
+ }
+
private async finalizeOwnedJobsForShutdown(): Promise<void> {
const ownedJobs = [...this.ownedJobs];
if (ownedJobs.length === 0) {
diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
index cd37e16f34..4e2e9e5c87 100644
--- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
@@ -1147,6 +1147,28 @@ describe('RedisEventTransport', () => {
}
});
+ it('persists provider drain proof under the exact generation and segment identity', async () => {
+ const mockPublisher = createMockPublisher();
+ const mockSubscriber = createMockSubscriber();
+ const transport = new RedisEventTransport(
+ mockPublisher as unknown as Redis,
+ mockSubscriber as unknown as Redis,
+ );
+ const streamId = 'provider-drain-proof';
+
+ await expect(transport.recordProviderDrain(streamId, 1234, 'segment-a')).resolves.toBe(true);
+ expect(mockPublisher.set).toHaveBeenCalledWith(
+ `stream:{${streamId}}:provider-drain:1234:segment-a`,
+ '1',
+ 'EX',
+ 86400,
+ );
+ await expect(transport.hasProviderDrain(streamId, 1234, 'segment-a')).resolves.toBe(true);
+ await expect(transport.hasProviderDrain(streamId, 1234, 'segment-b')).resolves.toBe(false);
+
+ transport.destroy();
+ });
+
it('waits for a delayed owner acknowledgement when cluster publish reports zero local receivers', async () => {
const mockPublisher = createMockPublisher();
const mockSubscriber = createMockSubscriber();
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
index d23931e963..7197f88aba 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
@@ -47,6 +47,54 @@ function jobHashFromCreationCall(call: unknown[]): Record<string, string> {
}
describe('RedisJobStore', () => {
+ test('marks only the exact provider segment drained', async () => {
+ const evalDrain = jest.fn().mockResolvedValue(1);
+ const redis = {
+ isCluster: true,
+ eval: evalDrain,
+ } as unknown as Cluster;
+ const store = new RedisJobStore(redis);
+
+ await expect(
+ store.markProviderExecutionDrained('stream-provider-drain', 123456, 'segment-a'),
+ ).resolves.toBe(true);
+
+ const [script, keyCount, jobKey, createdAt, providerExecutionId] = evalDrain.mock.calls[0];
+ expect(script).toContain('HGET", KEYS[1], "createdAt"');
+ expect(script).toContain('HGET", KEYS[1], "providerExecutionId"');
+ expect(script).toContain('HSET", KEYS[1], "providerDrained", "1"');
+ expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([
+ 1,
+ 'stream:{stream-provider-drain}:job',
+ '123456',
+ 'segment-a',
+ ]);
+ });
+
+ test('starts only the exact still-running initial provider segment', async () => {
+ const evalBegin = jest.fn().mockResolvedValue(1);
+ const redis = {
+ isCluster: true,
+ eval: evalBegin,
+ } as unknown as Cluster;
+ const store = new RedisJobStore(redis);
+
+ await expect(
+ store.beginProviderExecution('stream-provider-begin', 123456, 'segment-a'),
+ ).resolves.toBe(true);
+
+ const [script, keyCount, jobKey, createdAt, providerExecutionId] = evalBegin.mock.calls[0];
+ expect(script).toContain('HGET", KEYS[1], "status") ~= "running"');
+ expect(script).toContain('HGET", KEYS[1], "providerDrained") ~= "1"');
+ expect(script).toContain('HSET", KEYS[1], "providerDrained", "0"');
+ expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([
+ 1,
+ 'stream:{stream-provider-begin}:job',
+ '123456',
+ 'segment-a',
+ ]);
+ });
+
test('guards the atomic status transition with the expected creation epoch', async () => {
const evalTransition = jest.fn().mockResolvedValue(0);
const redis = {
@@ -411,7 +459,18 @@ describe('RedisJobStore', () => {
test('atomically resets predecessor state when creating a replacement', async () => {
const evalJobCreation = jest
.fn()
- .mockImplementation((...args: unknown[]) => ['user-1', '', args[Number(args[1]) + 3]]);
+ .mockImplementation((...args: unknown[]) => [
+ 'user-1',
+ '',
+ args[Number(args[1]) + 3],
+ '',
+ '1',
+ 'running',
+ 'conversation-before-replacement',
+ '1',
+ 'provider-before-replacement',
+ '0',
+ ]);
const redis = {
isCluster: true,
eval: evalJobCreation,
@@ -423,7 +482,16 @@ describe('RedisJobStore', () => {
const store = new RedisJobStore(redis);
store.setCollectedUsage('stream-replacement', [{ input_tokens: 10 }]);
- await store.createJob('stream-replacement', 'user-1');
+ const replacement = await store.createJob('stream-replacement', 'user-1');
+
+ expect(replacement.replacedJob).toMatchObject({
+ createdAt: 1,
+ status: 'running',
+ conversationId: 'conversation-before-replacement',
+ providerAbortReady: true,
+ providerExecutionId: 'provider-before-replacement',
+ providerDrained: false,
+ });
const [script, keyCount, ...args] = evalJobCreation.mock.calls[0];
expect(script).toContain(
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
index af05fea461..18777ca2e2 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
@@ -138,7 +138,13 @@ describe('RedisJobStore Integration Tests', () => {
const store = new RedisJobStore(ioredisClient);
await store.initialize();
const streamId = `test-replaced-owner-${Date.now()}`;
- const predecessor = await store.createJob(streamId, 'user-1', streamId);
+ const providerExecutionId = 'provider-before-replacement';
+ const predecessor = await store.createJob(streamId, 'user-1', streamId, undefined, {
+ providerExecutionId,
+ });
+ await expect(
+ store.beginProviderExecution(streamId, predecessor.createdAt, providerExecutionId),
+ ).resolves.toBe(true);
await store.transitionStatus(streamId, {
from: 'running',
to: 'requires_action',
@@ -155,12 +161,20 @@ describe('RedisJobStore Integration Tests', () => {
status: 'requires_action',
conversationId: streamId,
});
+ expect(replacement.replacedJob).toMatchObject({
+ providerExecutionId,
+ providerDrained: false,
+ });
// The receipt is durably reconstructed for lost-reply recovery, but it
// remains non-enumerable so ordinary job serializers cannot expose it.
const durableReplacement = await store.getJob(streamId);
expect((durableReplacement as typeof replacement).replacedJob).toEqual(
replacement.replacedJob,
);
+ expect((durableReplacement as typeof replacement).replacedJob).toMatchObject({
+ providerExecutionId,
+ providerDrained: false,
+ });
expect(Object.getOwnPropertyDescriptor(durableReplacement!, 'replacedJob')).toMatchObject({
enumerable: false,
});
diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.ts
index 1ebf2c7064..7c7b3da8d0 100644
--- a/packages/api/src/stream/__tests__/startup.spec.ts
+++ b/packages/api/src/stream/__tests__/startup.spec.ts
@@ -137,6 +137,8 @@ describe('GenerationJobManager startup telemetry', () => {
isTemporary: false,
promptTokens: 0,
discoveredTools: [],
+ providerExecutionId: expect.any(String),
+ providerDrained: true,
});
expect(job.metadata.tenantId).toBeUndefined();
expect(job.metadata.pendingAction).toBeUndefined();
@@ -150,6 +152,176 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.destroy();
});
+ it('retains a terminal generation until its exact provider segment has drained', async () => {
+ const manager = createManager();
+ const streamId = 'stream-provider-drain-retention';
+ const job = await manager.createJob(streamId, 'user-1');
+ const providerExecutionId = job.metadata.providerExecutionId!;
+ await expect(
+ manager.beginProviderExecution(streamId, job.createdAt, providerExecutionId),
+ ).resolves.toBe(true);
+
+ await expect(manager.abortJob(streamId)).resolves.toMatchObject({ success: true });
+ await expect(manager.getActiveJobIdsForUser('user-1')).resolves.toEqual([]);
+ await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([streamId]);
+ await expect(manager.getJobStore().getJob(streamId)).resolves.toMatchObject({
+ status: 'aborted',
+ providerExecutionId,
+ providerDrained: false,
+ });
+
+ await expect(
+ manager.markProviderExecutionDrained(streamId, job.createdAt, 'different-segment'),
+ ).resolves.toBe(false);
+ await expect(manager.getJobStore().getJob(streamId)).resolves.toMatchObject({
+ providerDrained: false,
+ });
+
+ await expect(
+ manager.markProviderExecutionDrained(streamId, job.createdAt, providerExecutionId),
+ ).resolves.toBe(true);
+ await expect(manager.getJobStore().getJob(streamId)).resolves.toBeNull();
+ await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([]);
+ await manager.destroy();
+ });
+
+ it('keeps an undrained stale approval visible to destructive cleanup only', async () => {
+ const manager = createManager();
+ const streamId = 'stream-provider-drain-stale-approval';
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.beginProviderExecution(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
+ await manager.getJobStore().transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ });
+
+ await expect(manager.getActiveJobIdsForUser('user-1')).resolves.toEqual([]);
+ await expect(manager.getCleanupBlockingJobIdsForUser('user-1')).resolves.toEqual([streamId]);
+ await manager.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
+ await manager.destroy();
+ });
+
+ it('waits for provider-owner drain proof before destructive abort returns', async () => {
+ const manager = createManager();
+ const streamId = 'stream-provider-drain-wait';
+ const job = await manager.createJob(streamId, 'user-1');
+ await expect(
+ manager.beginProviderExecution(streamId, job.createdAt, job.metadata.providerExecutionId!),
+ ).resolves.toBe(true);
+ let abortSettled = false;
+ const aborting = manager
+ .abortJob(streamId, { awaitProviderDrain: true })
+ .finally(() => (abortSettled = true));
+
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(abortSettled).toBe(false);
+
+ await manager.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
+ await expect(aborting).resolves.toMatchObject({ success: true });
+ expect(abortSettled).toBe(true);
+ await manager.destroy();
+ });
+
+ it('finishes a destructive abort immediately when the provider never started', async () => {
+ const manager = createManager();
+ const streamId = 'stream-provider-never-started';
+ await manager.createJob(streamId, 'user-1');
+
+ await expect(manager.abortJob(streamId, { awaitProviderDrain: true })).resolves.toMatchObject({
+ success: true,
+ });
+ await expect(manager.getJobStore().getJob(streamId)).resolves.toBeNull();
+ await manager.destroy();
+ });
+
+ it('rechecks provider drain when provider startup races the destructive abort', async () => {
+ const manager = createManager();
+ const streamId = 'stream-provider-drain-start-race';
+ const job = await manager.createJob(streamId, 'user-1');
+ const jobStore = manager.getJobStore();
+ const transition = jobStore.transitionStatusAndDrainSteers.bind(jobStore);
+ jest
+ .spyOn(jobStore, 'transitionStatusAndDrainSteers')
+ .mockImplementationOnce(async (...args) => {
+ await expect(
+ manager.beginProviderExecution(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ ),
+ ).resolves.toBe(true);
+ return transition(...args);
+ });
+
+ let abortSettled = false;
+ const aborting = manager
+ .abortJob(streamId, { awaitProviderDrain: true })
+ .finally(() => (abortSettled = true));
+
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(abortSettled).toBe(false);
+
+ await manager.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
+ await expect(aborting).resolves.toMatchObject({ success: true });
+ expect(abortSettled).toBe(true);
+ await manager.destroy();
+ });
+
+ it('does not expose a replacement until its overwritten provider segment drains', async () => {
+ const manager = createManager();
+ const streamId = 'stream-provider-drain-replacement';
+ const predecessor = await manager.createJob(streamId, 'user-1');
+ await expect(
+ manager.beginProviderExecution(
+ streamId,
+ predecessor.createdAt,
+ predecessor.metadata.providerExecutionId!,
+ ),
+ ).resolves.toBe(true);
+ let replacementSettled = false;
+ const replacing = manager
+ .createJob(streamId, 'user-1')
+ .finally(() => (replacementSettled = true));
+
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(predecessor.abortController.signal.aborted).toBe(true);
+ expect(replacementSettled).toBe(false);
+
+ // The replacement already owns the job hash, so the predecessor can only
+ // publish its exact cross-replica proof; it must not mark the new segment.
+ await expect(
+ manager.markProviderExecutionDrained(
+ streamId,
+ predecessor.createdAt,
+ predecessor.metadata.providerExecutionId!,
+ ),
+ ).resolves.toBe(false);
+
+ const replacement = await replacing;
+ expect(replacement.createdAt).not.toBe(predecessor.createdAt);
+ expect(replacementSettled).toBe(true);
+ await manager.destroy();
+ });
+
it('clears omitted per-turn identity when an in-memory job is replaced', async () => {
const manager = createManager();
await manager.createJob('stream-replaced-metadata', 'user-1', 'conversation-1', {
@@ -703,6 +875,11 @@ describe('GenerationJobManager startup telemetry', () => {
eventTransport.emitChunk(streamId, { data: { label: 'mismatched' } }, job.createdAt + 1);
await expect(manager.abortJob(streamId)).resolves.toMatchObject({ success: true });
+ await manager.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
await expect(jobStore.getJob(streamId)).resolves.toBeNull();
expect(order).toEqual([]);
@@ -2162,6 +2339,11 @@ describe('GenerationJobManager startup telemetry', () => {
});
expect(job.abortController.signal.aborted).toBe(true);
expect(abortDisposer).toHaveBeenCalledTimes(1);
+ await manager.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
await expect(jobStore.getJob(streamId)).resolves.toBeNull();
await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([]);
} finally {
@@ -2890,6 +3072,11 @@ describe('GenerationJobManager startup telemetry', () => {
expect.objectContaining({ aborted: true }),
job.createdAt,
);
+ await manager.markProviderExecutionDrained(
+ streamId,
+ job.createdAt,
+ job.metadata.providerExecutionId!,
+ );
await expect(jobStore.getJob(streamId)).resolves.toBeNull();
expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
await manager.destroy();
diff --git a/packages/api/src/stream/implementations/InMemoryEventTransport.ts b/packages/api/src/stream/implementations/InMemoryEventTransport.ts
index f9e953b9da..522e63fed4 100644
--- a/packages/api/src/stream/implementations/InMemoryEventTransport.ts
+++ b/packages/api/src/stream/implementations/InMemoryEventTransport.ts
@@ -13,6 +13,18 @@ interface StreamState {
*/
export class InMemoryEventTransport implements IEventTransport {
private streams = new Map<string, StreamState>();
+ private providerDrainProofs = new Map<string, number>();
+ private providerDrainWrites = 0;
+
+ private static readonly PROVIDER_DRAIN_PROOF_TTL_MS = 5 * 60_000;
+
+ private providerDrainKey(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): string {
+ return `${streamId}:${generationId}:${providerExecutionId}`;
+ }
private getOrCreateStream(streamId: string): StreamState {
let state = this.streams.get(streamId);
@@ -110,6 +122,44 @@ export class InMemoryEventTransport implements IEventTransport {
}
}
+ async recordProviderDrain(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ const now = Date.now();
+ this.providerDrainWrites++;
+ if (this.providerDrainWrites % 256 === 0) {
+ for (const [key, expiresAt] of this.providerDrainProofs) {
+ if (expiresAt <= now) {
+ this.providerDrainProofs.delete(key);
+ }
+ }
+ }
+ this.providerDrainProofs.set(
+ this.providerDrainKey(streamId, generationId, providerExecutionId),
+ now + InMemoryEventTransport.PROVIDER_DRAIN_PROOF_TTL_MS,
+ );
+ return true;
+ }
+
+ async hasProviderDrain(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ const key = this.providerDrainKey(streamId, generationId, providerExecutionId);
+ const expiresAt = this.providerDrainProofs.get(key);
+ if (expiresAt == null) {
+ return false;
+ }
+ if (expiresAt <= Date.now()) {
+ this.providerDrainProofs.delete(key);
+ return false;
+ }
+ return true;
+ }
+
getSubscriberCount(streamId: string): number {
const state = this.streams.get(streamId);
return state?.emitter.listenerCount('chunk') ?? 0;
@@ -186,6 +236,8 @@ export class InMemoryEventTransport implements IEventTransport {
state.emitter.removeAllListeners();
}
this.streams.clear();
+ this.providerDrainProofs.clear();
+ this.providerDrainWrites = 0;
logger.debug('[InMemoryEventTransport] Destroyed');
}
}
diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts
index fc6bb4c8ed..a140491597 100644
--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts
+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts
@@ -285,6 +285,15 @@ export class InMemoryJobStore implements IJobStoreV2 {
) {
throw new Error('Invalid expected generation predecessor');
}
+ const providerExecutionId = initialMetadata.providerExecutionId;
+ if (
+ providerExecutionId != null &&
+ (providerExecutionId.length === 0 || providerExecutionId.length > 128)
+ ) {
+ throw new Error('Invalid provider execution id');
+ }
+ const safeInitialMetadata = { ...initialMetadata };
+ delete safeInitialMetadata.providerDrained;
const assertOwnerCompatible = (): void => {
const existingJob = this.jobs.get(streamId);
if (
@@ -467,11 +476,23 @@ export class InMemoryJobStore implements IJobStoreV2 {
enumerable: false,
});
}
+ if (previousJob.providerExecutionId != null) {
+ Object.defineProperty(replaced, 'providerExecutionId', {
+ value: previousJob.providerExecutionId,
+ enumerable: false,
+ });
+ }
+ if (previousJob.providerDrained != null) {
+ Object.defineProperty(replaced, 'providerDrained', {
+ value: previousJob.providerDrained,
+ enumerable: false,
+ });
+ }
addReplacedJob(replaced);
}
this.lastGenerationEpoch = createdAt;
const job: CreatedJobData = {
- ...initialMetadata,
+ ...safeInitialMetadata,
streamId,
userId,
...(tenantId && { tenantId }),
@@ -485,6 +506,7 @@ export class InMemoryJobStore implements IJobStoreV2 {
...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),
...(recoveredSteerId !== undefined && { recoveredSteerId }),
providerAbortReady: false,
+ ...(providerExecutionId != null && { providerDrained: true }),
syncSent: false,
};
if (creationAttemptId != null) {
@@ -661,6 +683,37 @@ export class InMemoryJobStore implements IJobStoreV2 {
Object.assign(job, updates);
}
+ async markProviderExecutionDrained(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (job?.createdAt !== expectedCreatedAt || job.providerExecutionId !== providerExecutionId) {
+ return false;
+ }
+ job.providerDrained = true;
+ return true;
+ }
+
+ async beginProviderExecution(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (
+ job?.createdAt !== expectedCreatedAt ||
+ job.providerExecutionId !== providerExecutionId ||
+ job.status !== 'running' ||
+ job.providerDrained !== true
+ ) {
+ return false;
+ }
+ job.providerDrained = false;
+ return true;
+ }
+
async finalizeTerminalPersistence(
streamId: string,
expectedCreatedAt: number,
@@ -1174,6 +1227,18 @@ export class InMemoryJobStore implements IJobStoreV2 {
* Also performs self-healing cleanup: removes stale entries for jobs that no longer exist.
*/
async getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {
+ return this.getJobIdsByUser(userId, tenantId, false);
+ }
+
+ async getCleanupBlockingJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {
+ return this.getJobIdsByUser(userId, tenantId, true);
+ }
+
+ private getJobIdsByUser(
+ userId: string,
+ tenantId: string | undefined,
+ includeUndrained: boolean,
+ ): string[] {
const userKey = tenantId ? `${tenantId}:${userId}` : userId;
const trackedIds = this.userJobMap.get(userKey);
if (!trackedIds || trackedIds.size === 0) {
@@ -1196,12 +1261,20 @@ export class InMemoryJobStore implements IJobStoreV2 {
// A pending-approval job still occupies the user's conversation slot — but
// only while its prompt is live: a past-`expiresAt` approval no longer
// counts as active (cleanup/expiry will finalize it).
- if (job.status === 'running' || job.status === 'requires_action') {
- if (job.status === 'requires_action' && isPendingActionStale(job)) {
+ if (
+ job.status === 'running' ||
+ job.status === 'requires_action' ||
+ (includeUndrained && job.providerDrained === false)
+ ) {
+ if (
+ job.status === 'requires_action' &&
+ isPendingActionStale(job) &&
+ !(includeUndrained && job.providerDrained === false)
+ ) {
continue;
}
activeIds.push(streamId);
- } else {
+ } else if (job.providerDrained !== false) {
// Self-healing: job completed/deleted but mapping wasn't cleaned - fix it now
trackedIds.delete(streamId);
}
diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.ts
index d172fb388c..c03d966eef 100644
--- a/packages/api/src/stream/implementations/RedisEventTransport.ts
+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts
@@ -36,6 +36,9 @@ const KEYS = {
/** Owner-issued proof that this exact generation processed an abort. */
abortAck: (streamId: string, generationId: number) =>
`stream:{${streamId}}:abort-ack:${generationId}`,
+ /** Exact proof that a provider segment has fully unwound. */
+ providerDrain: (streamId: string, generationId: number, providerExecutionId: string) =>
+ `stream:{${streamId}}:provider-drain:${generationId}:${providerExecutionId}`,
};
/**
@@ -206,6 +209,7 @@ const MAX_BUFFER_SIZE = 100;
const GENERATION_EPOCH_GRACE_TTL_SECONDS = 300;
/** Durable owner proof outlives receipt retries and process-local subscriptions. */
const ABORT_ACK_TTL_SECONDS = 86400;
+const PROVIDER_DRAIN_TTL_SECONDS = 86400;
/**
* Subscriber state for a stream
@@ -1270,6 +1274,37 @@ export class RedisEventTransport implements IEventTransport {
}
}
+ async recordProviderDrain(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ try {
+ await this.publisher.set(
+ KEYS.providerDrain(streamId, generationId, providerExecutionId),
+ '1',
+ 'EX',
+ PROVIDER_DRAIN_TTL_SECONDS,
+ );
+ return true;
+ } catch (error) {
+ logger.error(`[RedisEventTransport] Failed to persist provider drain proof:`, error);
+ return false;
+ }
+ }
+
+ async hasProviderDrain(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ return (
+ (await this.publisher.get(
+ KEYS.providerDrain(streamId, generationId, providerExecutionId),
+ )) === '1'
+ );
+ }
+
private async publishAbortAcknowledgement(
streamId: string,
generationId: number,
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index 2940a864c9..ec39812290 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -329,7 +329,8 @@ const REPLACEMENT_RECEIPT_ACK_LUA =
* ...hsetPairs]
* Returns: [previousUserId | "", previousTenantId | "", createdAt, "",
* replacedCreatedAt | "", replacedStatus | "", replacedConversationId | "",
- * replacedProviderAbortReady | ""]
+ * replacedProviderAbortReady | "", replacedProviderExecutionId | "",
+ * replacedProviderDrained | ""]
* Predecessor mismatch returns the latest retained epoch in the replaced
* position plus an eighth active flag and ninth verified flag ("1" | "0").
* Job-only metadata is empty when that epoch has outlived its hash. When all
@@ -351,6 +352,8 @@ const JOB_CREATE_LUA =
'local replacedStatus = redis.call("HGET", KEYS[1], "status") ' +
'local replacedConversationId = redis.call("HGET", KEYS[1], "conversationId") ' +
'local replacedProviderAbortReady = redis.call("HGET", KEYS[1], "providerAbortReady") ' +
+ 'local replacedProviderExecutionId = redis.call("HGET", KEYS[1], "providerExecutionId") ' +
+ 'local replacedProviderDrained = redis.call("HGET", KEYS[1], "providerDrained") ' +
'local replacedProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") ' +
'local MAX_SAFE_EPOCH = 9007199254740991 ' +
'local function isSafeEpoch(value) return type(value) == "number" and value >= 0 ' +
@@ -360,6 +363,12 @@ const JOB_CREATE_LUA =
'local replacedEpoch = tonumber(replacedCreatedAt) local previousCreatedAt = replacedEpoch ' +
'if previousJobExists == 1 and (not isSafeEpoch(replacedEpoch) or not isValidJobStatus(replacedStatus)) then ' +
'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'if (replacedProviderExecutionId and not replacedProviderDrained) ' +
+ 'or (replacedProviderDrained and not replacedProviderExecutionId) ' +
+ 'or (replacedProviderExecutionId and (replacedProviderExecutionId == "" ' +
+ 'or string.len(replacedProviderExecutionId) > 128)) ' +
+ 'or (replacedProviderDrained and replacedProviderDrained ~= "0" and replacedProviderDrained ~= "1") then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
'local retainedEpochRaw = redis.call("GET", KEYS[7]) local retainedEpoch = tonumber(retainedEpochRaw) ' +
'if retainedEpochRaw and not isSafeEpoch(retainedEpoch) then ' +
'return { "", "", "0", "generation_epoch_corrupt" } end ' +
@@ -406,6 +415,10 @@ const JOB_CREATE_LUA =
'or (previousJobExists == 1 and item.createdAt >= replacedEpoch) ' +
'or (item.conversationId and type(item.conversationId) ~= "string") ' +
'or (item.providerAbortReady ~= nil and type(item.providerAbortReady) ~= "boolean") ' +
+ 'or (item.providerExecutionId ~= nil and (type(item.providerExecutionId) ~= "string" ' +
+ 'or item.providerExecutionId == "" or string.len(item.providerExecutionId) > 128)) ' +
+ 'or (item.providerDrained ~= nil and type(item.providerDrained) ~= "boolean") ' +
+ 'or ((item.providerExecutionId ~= nil) ~= (item.providerDrained ~= nil)) ' +
'or replacementSeen[tostring(item.createdAt)] then ' +
'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
'lastReplacementEpoch = item.createdAt replacementSeen[tostring(item.createdAt)] = true ' +
@@ -430,6 +443,8 @@ const JOB_CREATE_LUA =
'local replaced = { createdAt = replacedEpoch, status = replacedStatus } ' +
'if replacedConversationId then replaced.conversationId = replacedConversationId end ' +
'if replacedProviderAbortReady then replaced.providerAbortReady = replacedProviderAbortReady == "1" end ' +
+ 'if replacedProviderExecutionId then replaced.providerExecutionId = replacedProviderExecutionId end ' +
+ 'if replacedProviderDrained then replaced.providerDrained = replacedProviderDrained == "1" end ' +
'replacementChain[#replacementChain + 1] = replaced replacementSeen[tostring(replacedEpoch)] = true end ' +
'local recoveredSteerId = ARGV[5] local expectedRecovery = nil ' +
'if recoveredSteerId ~= "" and ARGV[10] ~= "2" then return { "", "", "0", "recovery_payload_mismatch" } end ' +
@@ -533,7 +548,8 @@ const JOB_CREATE_LUA =
'if claimTtl > 0 then redis.call("PEXPIRE", KEYS[10], claimTtl) end end ' +
'return { previousUserId or "", previousTenantId or "", tostring(createdAt), "", ' +
'replacedCreatedAt or "", replacedStatus or "", replacedConversationId or "", ' +
- 'replacedProviderAbortReady or "" }';
+ 'replacedProviderAbortReady or "", replacedProviderExecutionId or "", ' +
+ 'replacedProviderDrained or "" }';
/**
* Epoch-guarded field update. Terminal writes reclaim same-slot content in the
@@ -567,6 +583,23 @@ const JOB_UPDATE_LUA =
'end ' +
'return 1';
+/** Exact provider-segment completion fence. A paused segment finishing after a
+ * resume cannot mark the resumed provider drained because its opaque id differs. */
+const PROVIDER_DRAIN_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "providerExecutionId") ~= ARGV[2] then return 0 end ' +
+ 'redis.call("HSET", KEYS[1], "providerDrained", "1") return 1';
+
+/** Exact initial provider-start fence. The controller rechecks account
+ * deletion before this CAS; an abort/replacement that wins next prevents the
+ * provider from starting after destructive cleanup has begun. */
+const PROVIDER_BEGIN_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "providerExecutionId") ~= ARGV[2] then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "providerDrained") ~= "1" then return 0 end ' +
+ 'redis.call("HSET", KEYS[1], "providerDrained", "0") return 1';
+
/** Single-winner promotion from abort-persistence pending to a consumable
* terminal payload. Owner success/failure and stale-owner recovery share this
* CAS, so a timeout cannot overwrite a normal FINAL (or vice versa). */
@@ -1757,6 +1790,15 @@ export class RedisJobStore implements IJobStoreV2 {
) {
throw new Error('Invalid expected generation predecessor');
}
+ const providerExecutionId = initialMetadata.providerExecutionId;
+ if (
+ providerExecutionId != null &&
+ (providerExecutionId.length === 0 || providerExecutionId.length > 128)
+ ) {
+ throw new Error('Invalid provider execution id');
+ }
+ const safeInitialMetadata = { ...initialMetadata };
+ delete safeInitialMetadata.providerDrained;
let generationProtocolVersion: 1 | 2 = 1;
if (
initialMetadata.generationProtocolVersion === 1 ||
@@ -1767,7 +1809,7 @@ export class RedisJobStore implements IJobStoreV2 {
generationProtocolVersion = 2;
}
const job: CreatedJobData = {
- ...initialMetadata,
+ ...safeInitialMetadata,
streamId,
userId,
...(tenantId && { tenantId }),
@@ -1778,6 +1820,7 @@ export class RedisJobStore implements IJobStoreV2 {
...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),
...(recoveredSteerId !== undefined && { recoveredSteerId }),
providerAbortReady: false,
+ ...(providerExecutionId != null && { providerDrained: true }),
syncSent: false,
};
if (creationAttemptId != null) {
@@ -1909,6 +1952,18 @@ export class RedisJobStore implements IJobStoreV2 {
previousOwner[7] !== ''
? previousOwner[7] === '1'
: undefined;
+ const replacedProviderExecutionId =
+ Array.isArray(previousOwner) &&
+ typeof previousOwner[8] === 'string' &&
+ previousOwner[8] !== ''
+ ? previousOwner[8]
+ : undefined;
+ const replacedProviderDrained =
+ Array.isArray(previousOwner) &&
+ typeof previousOwner[9] === 'string' &&
+ previousOwner[9] !== ''
+ ? previousOwner[9] === '1'
+ : undefined;
const replacedJob =
replacedCreatedAt != null && Number.isFinite(replacedCreatedAt) && replacedStatus != null
? {
@@ -1925,6 +1980,18 @@ export class RedisJobStore implements IJobStoreV2 {
enumerable: false,
});
}
+ if (replacedJob != null && replacedProviderExecutionId != null) {
+ Object.defineProperties(replacedJob, {
+ providerExecutionId: {
+ value: replacedProviderExecutionId,
+ enumerable: false,
+ },
+ providerDrained: {
+ value: replacedProviderDrained,
+ enumerable: false,
+ },
+ });
+ }
const previousUserKeys =
previousUserId !== ''
? [KEYS.userJobs(previousUserId, previousTenantId || undefined)]
@@ -2074,6 +2141,42 @@ export class RedisJobStore implements IJobStoreV2 {
}
}
+ async markProviderExecutionDrained(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ return (
+ Number(
+ await this.redis.eval(
+ PROVIDER_DRAIN_LUA,
+ 1,
+ KEYS.job(streamId),
+ String(expectedCreatedAt),
+ providerExecutionId,
+ ),
+ ) === 1
+ );
+ }
+
+ async beginProviderExecution(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean> {
+ return (
+ Number(
+ await this.redis.eval(
+ PROVIDER_BEGIN_LUA,
+ 1,
+ KEYS.job(streamId),
+ String(expectedCreatedAt),
+ providerExecutionId,
+ ),
+ ) === 1
+ );
+ }
+
async finalizeTerminalPersistence(
streamId: string,
expectedCreatedAt: number,
@@ -2103,7 +2206,8 @@ export class RedisJobStore implements IJobStoreV2 {
left.createdAt === right.createdAt &&
left.status === right.status &&
left.userId === right.userId &&
- left.tenantId === right.tenantId
+ left.tenantId === right.tenantId &&
+ left.providerDrained === right.providerDrained
);
}
@@ -2119,7 +2223,10 @@ export class RedisJobStore implements IJobStoreV2 {
observedUserKeys: Set<string>,
): Promise<SerializableJobData | null> {
const statusKey = job ? this.statusSetKey(job.status) : null;
- const activeUserKey = job && statusKey != null ? KEYS.userJobs(job.userId, job.tenantId) : null;
+ const activeUserKey =
+ job && (statusKey != null || job.providerDrained === false)
+ ? KEYS.userJobs(job.userId, job.tenantId)
+ : null;
if (this.isCluster) {
const operations: Promise<unknown>[] = [
@@ -2813,6 +2920,18 @@ export class RedisJobStore implements IJobStoreV2 {
* @returns Array of conversation IDs with active jobs
*/
async getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {
+ return this.getJobIdsByUser(userId, tenantId, false);
+ }
+
+ async getCleanupBlockingJobIdsByUser(userId: string, tenantId?: string): Promise<string[]> {
+ return this.getJobIdsByUser(userId, tenantId, true);
+ }
+
+ private async getJobIdsByUser(
+ userId: string,
+ tenantId: string | undefined,
+ includeUndrained: boolean,
+ ): Promise<string[]> {
const userJobsKey = KEYS.userJobs(userId, tenantId);
const trackedIds = await this.redis.smembers(userJobsKey);
@@ -2832,8 +2951,18 @@ export class RedisJobStore implements IJobStoreV2 {
// polling and can complete.
const belongsToUser =
job?.userId === userId && (job.tenantId ?? undefined) === (tenantId ?? undefined);
- if (belongsToUser && job && (job.status === 'running' || job.status === 'requires_action')) {
- if (job.status === 'requires_action' && isPendingActionStale(job)) {
+ if (
+ belongsToUser &&
+ job &&
+ (job.status === 'running' ||
+ job.status === 'requires_action' ||
+ (includeUndrained && job.providerDrained === false))
+ ) {
+ if (
+ job.status === 'requires_action' &&
+ isPendingActionStale(job) &&
+ !(includeUndrained && job.providerDrained === false)
+ ) {
continue;
}
activeIds.push(streamId);
@@ -2851,8 +2980,14 @@ export class RedisJobStore implements IJobStoreV2 {
if (
currentBelongsToUser &&
currentJob &&
- (currentJob.status === 'running' || currentJob.status === 'requires_action') &&
- !(currentJob.status === 'requires_action' && isPendingActionStale(currentJob))
+ (currentJob.status === 'running' ||
+ currentJob.status === 'requires_action' ||
+ (includeUndrained && currentJob.providerDrained === false)) &&
+ !(
+ currentJob.status === 'requires_action' &&
+ isPendingActionStale(currentJob) &&
+ !(includeUndrained && currentJob.providerDrained === false)
+ )
) {
activeIds.push(streamId);
}
@@ -4345,6 +4480,8 @@ export class RedisJobStore implements IJobStoreV2 {
preemptCapable: data.preemptCapable != null ? data.preemptCapable === '1' : undefined,
providerAbortReady:
data.providerAbortReady != null ? data.providerAbortReady === '1' : undefined,
+ providerExecutionId: data.providerExecutionId || undefined,
+ providerDrained: data.providerDrained != null ? data.providerDrained === '1' : undefined,
titleEvent: data.titleEvent || undefined,
replayEvents: data.replayEvents || undefined,
contextUsage: data.contextUsage || undefined,
@@ -4407,6 +4544,12 @@ export class RedisJobStore implements IJobStoreV2 {
(candidate.conversationId != null && typeof candidate.conversationId !== 'string') ||
(candidate.providerAbortReady != null &&
typeof candidate.providerAbortReady !== 'boolean') ||
+ (candidate.providerExecutionId != null &&
+ (typeof candidate.providerExecutionId !== 'string' ||
+ candidate.providerExecutionId.length === 0 ||
+ candidate.providerExecutionId.length > 128)) ||
+ (candidate.providerDrained != null && typeof candidate.providerDrained !== 'boolean') ||
+ (candidate.providerExecutionId != null) !== (candidate.providerDrained != null) ||
seen.has(candidate.createdAt as number)
) {
throw new Error('Invalid generation replacement receipt');
@@ -4426,6 +4569,16 @@ export class RedisJobStore implements IJobStoreV2 {
enumerable: false,
});
}
+ if (candidate.providerExecutionId != null) {
+ Object.defineProperty(receipt, 'providerExecutionId', {
+ value: candidate.providerExecutionId,
+ enumerable: false,
+ });
+ Object.defineProperty(receipt, 'providerDrained', {
+ value: candidate.providerDrained,
+ enumerable: false,
+ });
+ }
replacedJobs.push(receipt);
}
const scalarEpoch = scalarEpochRaw != null ? Number(scalarEpochRaw) : undefined;
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index ef49e2b472..1f1a5590f8 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -107,6 +107,13 @@ export interface SerializableJobData {
* evidence and must be treated like true by replacement handoff. */
providerAbortReady?: boolean;
+ /** Opaque identity of the currently executing provider segment. A HITL resume
+ * replaces it so an earlier paused segment cannot acknowledge the new run. */
+ providerExecutionId?: string;
+ /** False while the identified provider segment can still mutate user data;
+ * true before provider startup and after the owner has fully unwound. */
+ providerDrained?: boolean;
+
/** Whether the user-message created event has been emitted */
createdEventEmitted?: boolean;
@@ -201,7 +208,12 @@ export interface SerializableJobData {
* reconstruct it without exposing it through normal job serialization. */
export type ReplacedGeneration = Pick<
SerializableJobData,
- 'createdAt' | 'status' | 'conversationId' | 'providerAbortReady'
+ | 'createdAt'
+ | 'status'
+ | 'conversationId'
+ | 'providerAbortReady'
+ | 'providerExecutionId'
+ | 'providerDrained'
>;
/** Latest generation epoch checked by a conditional create. A retained epoch
@@ -269,6 +281,8 @@ export type JobMetadataPatch = Partial<
| 'discoveredTools'
| 'activityPhaseSnapshot'
| 'preemptCapable'
+ | 'providerExecutionId'
+ | 'providerDrained'
| 'generationProtocolVersion'
| 'resolvedAskUserQuestions'
>
@@ -651,7 +665,6 @@ export interface IJobStore {
getJobCountByStatus(status: JobStatus): Promise<number>;
destroy(): Promise<void>;
getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]>;
-
setGraph(streamId: string, graph: StandardGraph, expectedCreatedAt?: number): void;
setContentParts(
streamId: string,
@@ -750,6 +763,23 @@ export interface IJobStoreV2 extends IJobStore {
expectedCreatedAt?: number,
): Promise<void>;
+ /** Atomically marks only the exact provider segment as fully unwound. */
+ markProviderExecutionDrained(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean>;
+
+ /** Activates the provider only while its exact generation is still running. */
+ beginProviderExecution(
+ streamId: string,
+ expectedCreatedAt: number,
+ providerExecutionId: string,
+ ): Promise<boolean>;
+
+ /** Includes terminal jobs whose provider still owns user-data writes. */
+ getCleanupBlockingJobIdsByUser(userId: string, tenantId?: string): Promise<string[]>;
+
/** Atomically replaces an abort persistence-pending marker with the one
* terminal payload that late subscribers may consume. Exactly one of the
* owner, its failure path, or stale-owner recovery wins. */
@@ -1277,6 +1307,18 @@ export interface IEventTransport {
* A delayed replacement can use the proof after the owner's listeners retire. */
recordAbortAcknowledgement?(streamId: string, generationId: number): Promise<boolean>;
+ /** Persist/read exact proof that a provider segment can no longer mutate user data. */
+ recordProviderDrain?(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): Promise<boolean>;
+ hasProviderDrain?(
+ streamId: string,
+ generationId: number,
+ providerExecutionId: string,
+ ): Promise<boolean>;
+
/** Publish a predecessor DONE only while the current job's opaque creation
* attempt still carries that predecessor in its durable receipt chain. */
emitReplacedDoneConfirmed?(
diff --git a/packages/api/src/stream/jobStoreCapabilities.ts b/packages/api/src/stream/jobStoreCapabilities.ts
index f3e1de0caf..2df4cb6bed 100644
--- a/packages/api/src/stream/jobStoreCapabilities.ts
+++ b/packages/api/src/stream/jobStoreCapabilities.ts
@@ -10,6 +10,9 @@ import type { IJobStore, IJobStoreV2 } from './interfaces/IJobStore';
*/
export const JOB_STORE_V2_REQUIRED_METHODS = [
'acknowledgeReplacedJobs',
+ 'markProviderExecutionDrained',
+ 'beginProviderExecution',
+ 'getCleanupBlockingJobIdsByUser',
'finalizeTerminalPersistence',
'transitionStatusAndDrainSteers',
'takeoverIdempotencyKey',
diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts
index 88ee943a8e..145df044b8 100644
--- a/packages/api/src/types/stream.ts
+++ b/packages/api/src/types/stream.ts
@@ -41,6 +41,10 @@ export interface GenerationJobMetadata {
activityPhaseSnapshot?: ActivityPhaseSnapshot;
/** See `SerializableJobData.preemptCapable`. */
preemptCapable?: boolean;
+ /** Exact provider segment whose completion gates destructive user cleanup. */
+ providerExecutionId?: string;
+ /** False only while that exact provider segment can still mutate user data. */
+ providerDrained?: boolean;
/** Terminal close has atomically stopped new steer acceptance, even if the
* final status CAS has not yet run. */
steersClosed?: boolean;
diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts
index fdb7fe28d9..9220c5f67a 100644
--- a/packages/data-schemas/src/index.ts
+++ b/packages/data-schemas/src/index.ts
@@ -32,6 +32,7 @@ export {
MAX_AUDIT_LOG_LIMIT,
MAX_AUDIT_VERIFY_ROWS,
MAX_TOOL_FAVORITES,
+ AgentTriggerDeliveryConflictError,
MCPAuthorityProofError,
MAX_MCP_AUTHORITY_TARGETS,
createMCPAuthorityBootRevision,
diff --git a/packages/data-schemas/src/methods/index.ts b/packages/data-schemas/src/methods/index.ts
index b5b3347bed..01468b7649 100644
--- a/packages/data-schemas/src/methods/index.ts
+++ b/packages/data-schemas/src/methods/index.ts
@@ -93,6 +93,11 @@ import {
type UpdateSkillResult,
type ValidationIssue,
} from './skill';
+import {
+ createAgentTriggerDeliveryMethods,
+ AgentTriggerDeliveryConflictError,
+ type AgentTriggerDeliveryMethods,
+} from './triggerDelivery';
import { createSkillSyncMethods, type SkillSyncMethods } from './skillSync';
import type {
SkillSyncStatusInput,
@@ -147,6 +152,7 @@ export {
};
export { AUDIT_SCHEMA_VERSION, MAX_AUDIT_EXPORT_ROWS, MAX_AUDIT_LOG_LIMIT, MAX_AUDIT_VERIFY_ROWS };
export { MAX_TOOL_FAVORITES };
+export { AgentTriggerDeliveryConflictError };
export type AllMethods = UserMethods &
SessionMethods &
@@ -182,6 +188,7 @@ export type AllMethods = UserMethods &
PromptMethods &
SkillMethods &
SkillSyncMethods &
+ AgentTriggerDeliveryMethods &
AgentMethods &
ConfigMethods &
MCPAuthorityMethods;
@@ -318,6 +325,7 @@ export function createMethods(
...promptMethods,
...skillMethods,
...createSkillSyncMethods(mongoose),
+ ...createAgentTriggerDeliveryMethods(mongoose),
/* Tier 5 */
...agentMethods,
/* Config */
@@ -375,6 +383,7 @@ export type {
SkillSyncCredentialSummary,
UpsertSkillSyncCredentialInput,
SkillSyncMethods,
+ AgentTriggerDeliveryMethods,
AgentMethods,
ConfigMethods,
MCPAuthorityMethods,
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
new file mode 100644
index 0000000000..6e1e2dc292
--- /dev/null
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -0,0 +1,956 @@
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import type { Model } from 'mongoose';
+import type {
+ AgentTriggerDeliveryFailure,
+ IAgentTriggerDeliveryDocument,
+ IAgentTriggerLaneSequenceDocument,
+ IAgentTriggerUserPurgeDocument,
+} from '~/types/triggerDelivery';
+import type { IUser } from '~/types/user';
+import {
+ AgentTriggerDeliveryConflictError,
+ createAgentTriggerDeliveryMethods,
+ type AgentTriggerDeliveryMethods,
+} from './triggerDelivery';
+import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';
+import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
+import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
+import { createUserModel } from '../models/user';
+
+jest.mock('~/config/winston', () => ({
+ error: jest.fn(),
+ warn: jest.fn(),
+ info: jest.fn(),
+ debug: jest.fn(),
+}));
+
+const DB_SETUP_TIMEOUT_MS = 60_000;
+const START = new Date('2026-08-17T12:00:00.000Z');
+let mongoServer: MongoMemoryServer;
+let Delivery: Model<IAgentTriggerDeliveryDocument>;
+let LaneSequence: Model<IAgentTriggerLaneSequenceDocument>;
+let UserPurge: Model<IAgentTriggerUserPurgeDocument>;
+let User: Model<IUser>;
+let methods: AgentTriggerDeliveryMethods;
+let counter = 0;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+ Delivery = createAgentTriggerDeliveryModel(mongoose);
+ LaneSequence = createAgentTriggerLaneSequenceModel(mongoose);
+ UserPurge = createAgentTriggerUserPurgeModel(mongoose);
+ User = createUserModel(mongoose);
+ await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);
+ methods = createAgentTriggerDeliveryMethods(mongoose);
+}, DB_SETUP_TIMEOUT_MS);
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+}, DB_SETUP_TIMEOUT_MS);
+
+beforeEach(async () => {
+ await Promise.all([
+ Delivery.deleteMany({}),
+ LaneSequence.deleteMany({}),
+ UserPurge.deleteMany({}),
+ User.deleteMany({}),
+ ]);
+});
+
+function enqueueInput(
+ overrides: Partial<Parameters<typeof methods.enqueueAgentTriggerDelivery>[0]> = {},
+) {
+ counter += 1;
+ return {
+ deliveryKey: `trigger_${counter}`,
+ fingerprint: `fingerprint_${counter}`,
+ orderingKey: 'ordering-1',
+ envelope: { version: 1, delivery: counter },
+ user: new mongoose.Types.ObjectId(),
+ tenantId: 'tenant-1',
+ availableAt: START,
+ ...overrides,
+ };
+}
+
+function transientFailure(overrides: Partial<AgentTriggerDeliveryFailure> = {}) {
+ return {
+ code: 'RATE_LIMITED',
+ message: 'try later',
+ certainty: 'definite' as const,
+ retryable: true,
+ attemptedAt: START,
+ status: 429,
+ ...overrides,
+ };
+}
+
+describe('agent trigger delivery methods', () => {
+ it('enqueues idempotently and rejects key reuse with different content', async () => {
+ const input = enqueueInput();
+ const first = await methods.enqueueAgentTriggerDelivery(input);
+ const replay = await methods.enqueueAgentTriggerDelivery(input);
+
+ expect(first.replayed).toBe(false);
+ expect(first.delivery).toMatchObject({ status: 'pending', attempts: 0 });
+ expect(replay).toMatchObject({ replayed: true, delivery: { id: first.delivery.id } });
+ await expect(
+ methods.enqueueAgentTriggerDelivery({ ...input, fingerprint: 'different' }),
+ ).rejects.toBeInstanceOf(AgentTriggerDeliveryConflictError);
+ expect(await Delivery.countDocuments()).toBe(1);
+ });
+
+ it('grants one atomic winner across concurrent claims', async () => {
+ await methods.enqueueAgentTriggerDelivery(enqueueInput());
+ const claims = await Promise.all(
+ Array.from({ length: 8 }, (_, index) =>
+ methods.claimNextAgentTriggerDelivery({
+ workerId: `worker-${index}`,
+ claimToken: `claim-${index}`,
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ }),
+ ),
+ );
+
+ expect(claims.filter((claim) => claim != null)).toHaveLength(1);
+ });
+
+ it('allocates a replica-stable monotonic sequence within an ordering lane', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const replicas = [
+ createAgentTriggerDeliveryMethods(mongoose),
+ createAgentTriggerDeliveryMethods(mongoose),
+ ];
+ const enqueued = await Promise.all(
+ Array.from({ length: 12 }, (_, index) =>
+ replicas[index % replicas.length].enqueueAgentTriggerDelivery(
+ enqueueInput({ user, orderingKey: 'shared-lane' }),
+ ),
+ ),
+ );
+
+ expect(enqueued.map(({ delivery }) => delivery.laneSequence).sort((a, b) => a - b)).toEqual(
+ Array.from({ length: 12 }, (_, index) => index + 1),
+ );
+ });
+
+ it('publishes an abandoned lane owner before allocating the next sequence', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'publication-race';
+ const firstInput = enqueueInput({ user, orderingKey, availableAt: START });
+ const first = await Delivery.create({
+ ...firstInput,
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+ await LaneSequence.create({
+ _id: orderingKey,
+ value: 1,
+ user,
+ tailDeliveryId: first._id,
+ publisherDeliveryId: first._id,
+ publisherStartedAt: START,
+ });
+
+ const second = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({ user, orderingKey, availableAt: START }),
+ );
+
+ await expect(Delivery.findById(first._id).lean()).resolves.toMatchObject({
+ status: 'pending',
+ laneSequence: 1,
+ });
+ expect(second.delivery).toMatchObject({ status: 'pending', laneSequence: 2 });
+ await expect(methods.findEarlierAgentTriggerDelivery(second.delivery)).resolves.toEqual({
+ availableAt: START,
+ });
+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({ value: 2 });
+ expect((await LaneSequence.findById(orderingKey).lean())?.publisherDeliveryId).toBeUndefined();
+ });
+
+ it('publishes an orphaned staging row before a later delivery in the lane', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'orphaned-staging';
+ const firstInput = enqueueInput({ user, orderingKey, availableAt: START });
+ const first = await Delivery.create({
+ ...firstInput,
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+
+ const second = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({ user, orderingKey, availableAt: START }),
+ );
+
+ await expect(Delivery.findById(first._id).lean()).resolves.toMatchObject({
+ status: 'pending',
+ laneSequence: 1,
+ });
+ expect(second.delivery).toMatchObject({ status: 'pending', laneSequence: 2 });
+ await expect(methods.findEarlierAgentTriggerDelivery(second.delivery)).resolves.toEqual({
+ availableAt: START,
+ });
+ });
+
+ it('discovers an orphaned staging row during maintenance recovery', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'maintenance-staging';
+ const staged = await Delivery.create({
+ ...enqueueInput({ user, orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ stagingRecoveryAt: START,
+ });
+
+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(1);
+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({
+ status: 'pending',
+ laneSequence: 1,
+ });
+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({ value: 1 });
+ });
+
+ it('backfills and recovers a staging row created before the fairness cursor existed', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'legacy-maintenance-staging';
+ const staged = await Delivery.create({
+ ...enqueueInput({ user, orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+
+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(1);
+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({
+ status: 'pending',
+ laneSequence: 1,
+ });
+ });
+
+ it('leaves staging unpublished while its durable user purge marker exists', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'purge-fenced-staging';
+ await UserPurge.create({ _id: user, fenceStartedAt: START, tenantId: 'tenant-1' });
+ const staged = await Delivery.create({
+ ...enqueueInput({ user, orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ stagingRecoveryAt: START,
+ });
+
+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);
+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({
+ status: 'staging',
+ laneSequence: 0,
+ });
+ await expect(LaneSequence.findById(orderingKey)).resolves.toBeNull();
+ });
+
+ it('leaves an existing lane publisher fenced while its user purge marker exists', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'purge-fenced-publisher';
+ await UserPurge.create({ _id: user, fenceStartedAt: START, tenantId: 'tenant-1' });
+ const staged = await Delivery.create({
+ ...enqueueInput({ user, orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+ await LaneSequence.create({
+ _id: orderingKey,
+ value: 1,
+ user,
+ tailDeliveryId: staged._id,
+ publisherDeliveryId: staged._id,
+ publisherStartedAt: START,
+ });
+
+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);
+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({
+ status: 'staging',
+ laneSequence: 0,
+ });
+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({
+ publisherDeliveryId: staged._id,
+ value: 1,
+ });
+ });
+
+ it('rotates purge-fenced staging rows so later recovery work is not starved', async () => {
+ const users = Array.from({ length: 3 }, () => new mongoose.Types.ObjectId());
+ for (let index = 0; index < users.length; index += 1) {
+ if (index < 2) {
+ await UserPurge.create({ _id: users[index], fenceStartedAt: START });
+ }
+ await Delivery.create({
+ ...enqueueInput({ user: users[index], orderingKey: `staging-rotation-${index}` }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ stagingRecoveryAt: new Date(START.getTime() + index),
+ });
+ }
+
+ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(0);
+ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(1);
+ await expect(
+ Delivery.findOne({ orderingKey: 'staging-rotation-2' }).lean(),
+ ).resolves.toMatchObject({ status: 'pending', laneSequence: 1 });
+ });
+
+ it('rotates purge-fenced publishers so later recovery work is not starved', async () => {
+ const users = Array.from({ length: 3 }, () => new mongoose.Types.ObjectId());
+ for (let index = 0; index < users.length; index += 1) {
+ if (index < 2) {
+ await UserPurge.create({ _id: users[index], fenceStartedAt: START });
+ }
+ const orderingKey = `publisher-rotation-${index}`;
+ const staged = await Delivery.create({
+ ...enqueueInput({ user: users[index], orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ stagingRecoveryAt: START,
+ });
+ await LaneSequence.create({
+ _id: orderingKey,
+ value: 1,
+ user: users[index],
+ publisherDeliveryId: staged._id,
+ publisherStartedAt: new Date(START.getTime() + index),
+ });
+ }
+
+ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(0);
+ await expect(methods.recoverAgentTriggerLanePublications(2)).resolves.toBe(1);
+ await expect(
+ Delivery.findOne({ orderingKey: 'publisher-rotation-2' }).lean(),
+ ).resolves.toMatchObject({ status: 'pending', laneSequence: 1 });
+ });
+
+ it('abandons a lane acquired concurrently with a new purge marker', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'purge-race-staging';
+ const staged = await Delivery.create({
+ ...enqueueInput({ user, orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+ const purgeExists = jest
+ .spyOn(UserPurge, 'exists')
+ .mockResolvedValueOnce(null)
+ .mockResolvedValueOnce({ _id: user } as never);
+
+ try {
+ await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);
+ } finally {
+ purgeExists.mockRestore();
+ }
+ await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({
+ status: 'staging',
+ laneSequence: 0,
+ });
+ await expect(LaneSequence.findById(orderingKey)).resolves.toBeNull();
+ });
+
+ it('builds a sparse index for the periodic publisher recovery scan', async () => {
+ const [laneIndexes, deliveryIndexes] = await Promise.all([
+ LaneSequence.collection.indexes(),
+ Delivery.collection.indexes(),
+ ]);
+
+ expect(laneIndexes).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ key: { publisherStartedAt: 1 }, sparse: true }),
+ ]),
+ );
+ expect(deliveryIndexes).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ key: { stagingRecoveryAt: 1 }, sparse: true }),
+ expect.objectContaining({ key: { laneCleanupPendingAt: 1 }, sparse: true }),
+ ]),
+ );
+ });
+
+ it('publishes an idempotent replay on its persisted ordering lane', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const input = enqueueInput({ user, orderingKey: 'original-lane' });
+ await Delivery.create({
+ ...input,
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+
+ const replay = await methods.enqueueAgentTriggerDelivery({
+ ...input,
+ orderingKey: 'changed-lane',
+ });
+
+ expect(replay).toMatchObject({
+ replayed: true,
+ delivery: { orderingKey: 'original-lane', laneSequence: 1, status: 'pending' },
+ });
+ await expect(LaneSequence.findById('original-lane').lean()).resolves.toMatchObject({
+ value: 1,
+ });
+ await expect(LaneSequence.findById('changed-lane').lean()).resolves.toBeNull();
+ });
+
+ it('resumes its own abandoned publication without allocating a second sequence', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'publication-replay';
+ const input = enqueueInput({ user, orderingKey, availableAt: START });
+ const staged = await Delivery.create({
+ ...input,
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+ await LaneSequence.create({
+ _id: orderingKey,
+ value: 1,
+ user,
+ tailDeliveryId: staged._id,
+ publisherDeliveryId: staged._id,
+ publisherStartedAt: START,
+ });
+
+ const replay = await methods.enqueueAgentTriggerDelivery(input);
+
+ expect(replay).toMatchObject({ replayed: true, delivery: { laneSequence: 1 } });
+ await expect(LaneSequence.findById(orderingKey).lean()).resolves.toMatchObject({ value: 1 });
+ });
+
+ it('orders a lane by its atomic sequence instead of tied timestamps or process ObjectIds', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const common = {
+ orderingKey: 'tied-lane',
+ envelope: { version: 1 },
+ user,
+ status: 'pending' as const,
+ attempts: 0,
+ availableAt: START,
+ createdAt: START,
+ updatedAt: START,
+ };
+ await Delivery.create([
+ {
+ ...common,
+ _id: new mongoose.Types.ObjectId('ffffffffffffffffffffffff'),
+ deliveryKey: 'first-by-sequence',
+ fingerprint: 'first',
+ laneSequence: 1,
+ },
+ {
+ ...common,
+ _id: new mongoose.Types.ObjectId('000000000000000000000001'),
+ deliveryKey: 'second-by-sequence',
+ fingerprint: 'second',
+ laneSequence: 2,
+ },
+ ]);
+
+ await expect(
+ methods.findEarlierAgentTriggerDelivery({
+ orderingKey: 'tied-lane',
+ laneSequence: 2,
+ }),
+ ).resolves.toEqual({ availableAt: START });
+ });
+
+ it('honors the lease skew margin before takeover', async () => {
+ await methods.enqueueAgentTriggerDelivery(enqueueInput());
+ const first = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ expect(first).not.toBeNull();
+
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-2',
+ claimToken: 'claim-2',
+ now: new Date(START.getTime() + 70_000),
+ leaseUntil: new Date(START.getTime() + 130_000),
+ }),
+ ).resolves.toBeNull();
+
+ await expect(
+ methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-2',
+ claimToken: 'claim-2',
+ now: new Date(START.getTime() + 91_000),
+ leaseUntil: new Date(START.getTime() + 151_000),
+ }),
+ ).resolves.toMatchObject({ claimToken: 'claim-2', leaseBy: 'worker-2' });
+ });
+
+ it('fences every settlement against a stale claimant', async () => {
+ await methods.enqueueAgentTriggerDelivery(enqueueInput());
+ const first = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 10_000),
+ });
+ expect(first).not.toBeNull();
+ const second = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-2',
+ now: new Date(START.getTime() + 41_000),
+ leaseUntil: new Date(START.getTime() + 101_000),
+ });
+ expect(second).not.toBeNull();
+
+ await expect(
+ methods.completeAgentTriggerDelivery({
+ id: first!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: 1,
+ result: { conversationId: 'stale' },
+ settledAt: START,
+ }),
+ ).resolves.toBe(false);
+ await expect(
+ methods.beginAgentTriggerDeliveryAttempt({
+ id: second!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-2',
+ now: new Date(START.getTime() + 41_000),
+ }),
+ ).resolves.toBe(1);
+ });
+
+ it('blocks a due delivery behind an earlier pending item in its ordering lane', async () => {
+ const first = await methods.enqueueAgentTriggerDelivery(
+ enqueueInput({ availableAt: new Date(START.getTime() + 60_000) }),
+ );
+ await methods.enqueueAgentTriggerDelivery(enqueueInput());
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ expect(claimed?.id).not.toBe(first.delivery.id);
+
+ await expect(methods.findEarlierAgentTriggerDelivery(claimed!)).resolves.toEqual({
+ availableAt: new Date(START.getTime() + 60_000),
+ });
+ });
+
+ it('records retry and success outcomes and expires only successful rows', async () => {
+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput());
+ const first = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: first!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ });
+ await methods.retryAgentTriggerDelivery({
+ id: first!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: attempt!,
+ error: transientFailure(),
+ availableAt: new Date(START.getTime() + 1_000),
+ });
+
+ const second = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-2',
+ claimToken: 'claim-2',
+ now: new Date(START.getTime() + 1_000),
+ leaseUntil: new Date(START.getTime() + 61_000),
+ });
+ const secondAttempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: second!.id,
+ workerId: 'worker-2',
+ claimToken: 'claim-2',
+ now: new Date(START.getTime() + 1_000),
+ });
+ await methods.completeAgentTriggerDelivery({
+ id: second!.id,
+ workerId: 'worker-2',
+ claimToken: 'claim-2',
+ attempt: secondAttempt!,
+ result: { conversationId: 'conversation-1' },
+ settledAt: new Date(START.getTime() + 2_000),
+ });
+
+ const stored = await methods.getAgentTriggerDelivery(queued.delivery.deliveryKey);
+ expect(stored).toMatchObject({
+ status: 'succeeded',
+ attempts: 2,
+ result: { conversationId: 'conversation-1' },
+ history: [
+ { attempt: 1, outcome: 'retry', workerId: 'worker-1' },
+ { attempt: 2, outcome: 'succeeded', workerId: 'worker-2' },
+ ],
+ });
+ expect(stored?.expiresAt?.getTime()).toBe(
+ new Date(START.getTime() + 2_000).getTime() + 90 * 24 * 60 * 60_000,
+ );
+ });
+
+ it('reclaims an inactive high-cardinality lane after its tail succeeds', async () => {
+ const orderingKey = 'one-shot-resource-lane';
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ orderingKey }));
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: claimed!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ });
+
+ await expect(
+ methods.completeAgentTriggerDelivery({
+ id: claimed!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: attempt!,
+ result: { ok: true },
+ settledAt: START,
+ }),
+ ).resolves.toBe(true);
+
+ expect(await LaneSequence.countDocuments({ _id: orderingKey })).toBe(0);
+ });
+
+ it('rediscovers lane cleanup when success commits before its cleanup marker', async () => {
+ const orderingKey = 'crashed-lane-cleanup';
+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput({ orderingKey }));
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: claimed!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ });
+ const laneUpdate = jest
+ .spyOn(LaneSequence, 'updateOne')
+ .mockRejectedValueOnce(new Error('down'));
+
+ try {
+ await expect(
+ methods.completeAgentTriggerDelivery({
+ id: claimed!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: attempt!,
+ result: { ok: true },
+ settledAt: START,
+ }),
+ ).resolves.toBe(true);
+ } finally {
+ laneUpdate.mockRestore();
+ }
+ await expect(Delivery.findById(queued.delivery.id).lean()).resolves.toMatchObject({
+ status: 'succeeded',
+ laneCleanupPendingAt: START,
+ });
+
+ await expect(methods.reclaimInactiveAgentTriggerLanes(1)).resolves.toBe(1);
+ await expect(LaneSequence.findById(orderingKey)).resolves.toBeNull();
+ expect(
+ (await Delivery.findById(queued.delivery.id).lean())?.laneCleanupPendingAt,
+ ).toBeUndefined();
+ });
+
+ it('restores the retry budget when dispatch is deferred before execution', async () => {
+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput());
+ const claim = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: claim!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ });
+
+ await expect(
+ methods.deferAgentTriggerDeliveryAttempt({
+ id: claim!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: attempt!,
+ availableAt: new Date(START.getTime() + 5_000),
+ }),
+ ).resolves.toBe(true);
+
+ await expect(
+ methods.getAgentTriggerDelivery(queued.delivery.deliveryKey),
+ ).resolves.toMatchObject({
+ status: 'pending',
+ attempts: 0,
+ availableAt: new Date(START.getTime() + 5_000),
+ });
+ });
+
+ it('retains dead letters, truncates failure text, and supports explicit requeue', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const orderingKey = 'dead-letter-requeue';
+ const queued = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user, orderingKey }));
+ const claim = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ const attempt = await methods.beginAgentTriggerDeliveryAttempt({
+ id: claim!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ });
+ await methods.deadLetterAgentTriggerDelivery({
+ id: claim!.id,
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ attempt: attempt!,
+ error: transientFailure({
+ code: 'X'.repeat(256),
+ message: 'x'.repeat(3_000),
+ retryable: false,
+ }),
+ settledAt: START,
+ });
+
+ const [dead] = await methods.getAgentTriggerDeadLetters();
+ expect(dead).toMatchObject({ id: queued.delivery.id, status: 'dead' });
+ expect(dead.lastError?.code).toHaveLength(128);
+ expect(dead.lastError?.message).toHaveLength(2048);
+ expect(dead.expiresAt).toBeUndefined();
+
+ const later = await methods.enqueueAgentTriggerDelivery(enqueueInput({ user, orderingKey }));
+ const laterClaim = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-2',
+ claimToken: 'claim-2',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ expect(laterClaim?.id).toBe(later.delivery.id);
+
+ const stagedAhead = await Delivery.create({
+ ...enqueueInput({ user, orderingKey }),
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ });
+ await Delivery.collection.updateOne(
+ { _id: stagedAhead._id },
+ { $set: { updatedAt: new Date(0) } },
+ );
+
+ const requeued = await methods.requeueAgentTriggerDelivery(dead.id, START);
+ expect(requeued).toMatchObject({
+ status: 'pending',
+ attempts: 0,
+ requeueCount: 1,
+ laneSequence: 4,
+ });
+ await expect(Delivery.findById(stagedAhead._id).lean()).resolves.toMatchObject({
+ status: 'pending',
+ laneSequence: 3,
+ });
+ expect(requeued?.lastError).toBeUndefined();
+ await expect(methods.findEarlierAgentTriggerDelivery(requeued!)).resolves.toEqual({
+ availableAt: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+ expect(await methods.getAgentTriggerDeadLetters()).toEqual([]);
+ await expect(methods.getAgentTriggerDeadLetters(Number.NaN)).rejects.toThrow(
+ 'Agent trigger dead-letter limit must be a positive integer',
+ );
+ });
+
+ it('counts only live leases while an account deletion drains', async () => {
+ const user = new mongoose.Types.ObjectId();
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'claim-1',
+ now: START,
+ leaseUntil: new Date(START.getTime() + 60_000),
+ });
+
+ expect(await methods.countActiveAgentTriggerDeliveriesByUser(user, START)).toBe(1);
+ expect(
+ await methods.countActiveAgentTriggerDeliveriesByUser(
+ user,
+ new Date(START.getTime() + 60_001),
+ ),
+ ).toBe(0);
+ });
+
+ it('deletes all queued payloads for an erased user', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const fenceStartedAt = new Date(START);
+ await User.create({
+ _id: user,
+ email: '[email protected]',
+ provider: 'local',
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ orderingKey: 'ordering-other' }));
+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);
+ await User.deleteOne({ _id: user });
+
+ await methods.deleteAgentTriggerDeliveriesByUser(user);
+
+ expect(await Delivery.countDocuments({ user })).toBe(0);
+ expect(await Delivery.countDocuments()).toBe(1);
+ expect(await LaneSequence.countDocuments({ user })).toBe(0);
+ expect(await LaneSequence.countDocuments()).toBe(1);
+ expect(await UserPurge.countDocuments({ _id: user })).toBe(0);
+ });
+
+ it('recovers an armed purge after the user deletion commits', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const fenceStartedAt = new Date(START);
+ await User.create({
+ _id: user,
+ email: '[email protected]',
+ provider: 'local',
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt, 'tenant-1');
+
+ await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(0);
+ expect(await Delivery.countDocuments({ user })).toBe(1);
+ expect(await UserPurge.countDocuments({ _id: user })).toBe(1);
+
+ await User.deleteOne({ _id: user });
+ await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(1);
+ expect(await Delivery.countDocuments({ user })).toBe(0);
+ expect(await LaneSequence.countDocuments({ user })).toBe(0);
+ expect(await UserPurge.countDocuments({ _id: user })).toBe(0);
+ });
+
+ it('rotates active purge markers so a later committed purge is recovered', async () => {
+ const users = Array.from({ length: 3 }, () => new mongoose.Types.ObjectId());
+ for (const [index, user] of users.entries()) {
+ await User.create({
+ _id: user,
+ email: `purge-rotation-${index}@example.com`,
+ provider: 'local',
+ agentTriggerDeletionStartedAt: START,
+ });
+ await methods.prepareAgentTriggerUserPurge(user, START);
+ }
+ await Delivery.create({
+ ...enqueueInput({ user: users[2], orderingKey: 'committed-purge-lane' }),
+ laneSequence: 1,
+ status: 'pending',
+ attempts: 0,
+ requeueCount: 0,
+ });
+ await User.deleteOne({ _id: users[2] });
+
+ await expect(methods.recoverAgentTriggerUserPurges(2)).resolves.toBe(0);
+ await expect(methods.recoverAgentTriggerUserPurges(2)).resolves.toBe(1);
+ expect(await UserPurge.countDocuments({ _id: users[2] })).toBe(0);
+ expect(await Delivery.countDocuments({ user: users[2] })).toBe(0);
+ expect(await UserPurge.countDocuments({ _id: { $in: users.slice(0, 2) } })).toBe(2);
+ });
+
+ it('disarms a stale purge marker without touching an active user delivery', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const fenceStartedAt = new Date(START);
+ await User.create({
+ _id: user,
+ email: '[email protected]',
+ provider: 'local',
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ await methods.enqueueAgentTriggerDelivery(enqueueInput({ user }));
+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);
+ await User.updateOne({ _id: user }, { $unset: { agentTriggerDeletionStartedAt: 1 } });
+
+ await expect(methods.recoverAgentTriggerUserPurges()).resolves.toBe(0);
+ expect(await Delivery.countDocuments({ user })).toBe(1);
+ expect(await UserPurge.countDocuments({ _id: user })).toBe(0);
+ });
+
+ it('lets only the exact deletion owner cancel a prepared purge', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const fenceStartedAt = new Date(START);
+ await User.create({
+ _id: user,
+ email: '[email protected]',
+ provider: 'local',
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);
+
+ await expect(
+ methods.cancelAgentTriggerUserPurge(user, new Date(START.getTime() + 1)),
+ ).resolves.toBe(false);
+ await expect(methods.cancelAgentTriggerUserPurge(user, fenceStartedAt)).resolves.toBe(true);
+ });
+
+ it('cannot disarm purge recovery after the user commit', async () => {
+ const user = new mongoose.Types.ObjectId();
+ const fenceStartedAt = new Date(START);
+ await User.create({
+ _id: user,
+ email: '[email protected]',
+ provider: 'local',
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ await methods.prepareAgentTriggerUserPurge(user, fenceStartedAt);
+ await User.deleteOne({ _id: user });
+
+ await expect(methods.cancelAgentTriggerUserPurge(user, fenceStartedAt)).resolves.toBe(false);
+ expect(await UserPurge.countDocuments({ _id: user })).toBe(1);
+ });
+});
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
new file mode 100644
index 0000000000..0e718caecc
--- /dev/null
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -0,0 +1,995 @@
+import type { Model, Types } from 'mongoose';
+import type {
+ AgentTriggerDeliveryClaim,
+ AgentTriggerDeliveryFailure,
+ AgentTriggerDeliveryRecord,
+ AgentTriggerOrderingBlock,
+ IAgentTriggerDelivery,
+ IAgentTriggerDeliveryDocument,
+ IAgentTriggerLaneSequence,
+ IAgentTriggerLaneSequenceDocument,
+ IAgentTriggerUserPurge,
+ IAgentTriggerUserPurgeDocument,
+} from '~/types/triggerDelivery';
+import { createIndexesWithRetry } from '~/utils/retry';
+import logger from '~/config/winston';
+
+const DUPLICATE_KEY = 11000;
+const LEASE_SKEW_MARGIN_MS = 30_000;
+const SUCCESS_RETENTION_MS = 90 * 24 * 60 * 60_000;
+const MAX_ERROR_CODE_LENGTH = 128;
+const MAX_ERROR_MESSAGE_LENGTH = 2048;
+const MAX_DEAD_LETTER_LIMIT = 200;
+const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
+const MAX_PURGE_RECOVERY_LIMIT = 200;
+const HISTORY_LIMIT = 64;
+
+type DuplicateKeyError = { code?: number };
+
+export class AgentTriggerDeliveryConflictError extends Error {
+ constructor(deliveryKey: string) {
+ super(`Agent trigger delivery key ${deliveryKey} was reused with different content`);
+ this.name = 'AgentTriggerDeliveryConflictError';
+ }
+}
+
+export interface EnqueueAgentTriggerDeliveryInput {
+ deliveryKey: string;
+ fingerprint: string;
+ orderingKey: string;
+ envelope: unknown;
+ user: string | Types.ObjectId;
+ tenantId?: string;
+ availableAt: Date;
+}
+
+export interface AgentTriggerDeliveryFence {
+ id: string;
+ workerId: string;
+ claimToken: string;
+}
+
+export interface AgentTriggerDeliveryMethods {
+ ensureAgentTriggerDeliveryIndexes: () => Promise<void>;
+ enqueueAgentTriggerDelivery: (
+ input: EnqueueAgentTriggerDeliveryInput,
+ ) => Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }>;
+ claimNextAgentTriggerDelivery: (input: {
+ workerId: string;
+ claimToken: string;
+ now: Date;
+ leaseUntil: Date;
+ }) => Promise<AgentTriggerDeliveryClaim | null>;
+ findEarlierAgentTriggerDelivery: (
+ delivery: Pick<AgentTriggerDeliveryRecord, 'orderingKey' | 'laneSequence'>,
+ ) => Promise<AgentTriggerOrderingBlock | null>;
+ releaseAgentTriggerDelivery: (
+ input: AgentTriggerDeliveryFence & { availableAt: Date },
+ ) => Promise<boolean>;
+ beginAgentTriggerDeliveryAttempt: (
+ input: AgentTriggerDeliveryFence & { now: Date },
+ ) => Promise<number | null>;
+ deferAgentTriggerDeliveryAttempt: (
+ input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },
+ ) => Promise<boolean>;
+ completeAgentTriggerDelivery: (
+ input: AgentTriggerDeliveryFence & {
+ attempt: number;
+ result: unknown;
+ settledAt: Date;
+ },
+ ) => Promise<boolean>;
+ retryAgentTriggerDelivery: (
+ input: AgentTriggerDeliveryFence & {
+ attempt: number;
+ error: AgentTriggerDeliveryFailure;
+ availableAt: Date;
+ },
+ ) => Promise<boolean>;
+ deadLetterAgentTriggerDelivery: (
+ input: AgentTriggerDeliveryFence & {
+ attempt: number;
+ error: AgentTriggerDeliveryFailure;
+ settledAt: Date;
+ },
+ ) => Promise<boolean>;
+ getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;
+ getAgentTriggerDeadLetters: (limit?: number) => Promise<AgentTriggerDeliveryRecord[]>;
+ requeueAgentTriggerDelivery: (
+ id: string,
+ availableAt: Date,
+ ) => Promise<AgentTriggerDeliveryRecord | null>;
+ countActiveAgentTriggerDeliveriesByUser: (
+ user: string | Types.ObjectId,
+ now: Date,
+ ) => Promise<number>;
+ recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;
+ reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;
+ prepareAgentTriggerUserPurge: (
+ user: string | Types.ObjectId,
+ fenceStartedAt: Date,
+ tenantId?: string,
+ ) => Promise<void>;
+ cancelAgentTriggerUserPurge: (
+ user: string | Types.ObjectId,
+ fenceStartedAt: Date,
+ ) => Promise<boolean>;
+ recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
+ deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;
+}
+
+function normalizeFailure(error: AgentTriggerDeliveryFailure): AgentTriggerDeliveryFailure {
+ return {
+ ...error,
+ code: error.code.slice(0, MAX_ERROR_CODE_LENGTH),
+ message: error.message.slice(0, MAX_ERROR_MESSAGE_LENGTH),
+ };
+}
+
+function toRecord(delivery: IAgentTriggerDelivery): AgentTriggerDeliveryRecord {
+ if (delivery._id == null || delivery.createdAt == null) {
+ throw new Error('Persisted agent trigger delivery is missing its identity or creation time');
+ }
+ return {
+ id: String(delivery._id),
+ deliveryKey: delivery.deliveryKey,
+ fingerprint: delivery.fingerprint,
+ orderingKey: delivery.orderingKey,
+ laneSequence: delivery.laneSequence,
+ envelope: delivery.envelope,
+ user: delivery.user,
+ status: delivery.status,
+ attempts: delivery.attempts,
+ availableAt: delivery.availableAt,
+ createdAt: delivery.createdAt,
+ ...(delivery.tenantId != null && { tenantId: delivery.tenantId }),
+ ...(delivery.leaseBy != null && { leaseBy: delivery.leaseBy }),
+ ...(delivery.leaseUntil != null && { leaseUntil: delivery.leaseUntil }),
+ ...(delivery.claimToken != null && { claimToken: delivery.claimToken }),
+ ...(delivery.lastError != null && { lastError: delivery.lastError }),
+ ...(delivery.result !== undefined && { result: delivery.result }),
+ ...(delivery.history != null && { history: delivery.history }),
+ ...(delivery.settledAt != null && { settledAt: delivery.settledAt }),
+ ...(delivery.expiresAt != null && { expiresAt: delivery.expiresAt }),
+ ...(delivery.requeueCount != null && { requeueCount: delivery.requeueCount }),
+ ...(delivery.updatedAt != null && { updatedAt: delivery.updatedAt }),
+ };
+}
+
+function requireClaim(delivery: IAgentTriggerDelivery | null): AgentTriggerDeliveryClaim | null {
+ if (delivery == null) {
+ return null;
+ }
+ const record = toRecord(delivery);
+ if (
+ record.status !== 'leased' ||
+ record.claimToken == null ||
+ record.leaseBy == null ||
+ record.leaseUntil == null
+ ) {
+ throw new Error('Claimed agent trigger delivery is missing its lease fence');
+ }
+ return record as AgentTriggerDeliveryClaim;
+}
+
+export function createAgentTriggerDeliveryMethods(
+ mongoose: typeof import('mongoose'),
+): AgentTriggerDeliveryMethods {
+ const Delivery = () =>
+ mongoose.models.AgentTriggerDelivery as Model<IAgentTriggerDeliveryDocument>;
+ const LaneSequence = () =>
+ mongoose.models.AgentTriggerLaneSequence as Model<IAgentTriggerLaneSequenceDocument>;
+ const UserPurge = () =>
+ mongoose.models.AgentTriggerUserPurge as Model<IAgentTriggerUserPurgeDocument>;
+
+ async function ensureAgentTriggerDeliveryIndexes(): Promise<void> {
+ await Promise.all([
+ createIndexesWithRetry(Delivery()),
+ createIndexesWithRetry(LaneSequence()),
+ createIndexesWithRetry(UserPurge()),
+ ]);
+ }
+
+ async function abandonLanePublisher(lane: IAgentTriggerLaneSequence): Promise<boolean> {
+ const publisherDeliveryId = lane.publisherDeliveryId;
+ if (publisherDeliveryId == null) {
+ return false;
+ }
+ if (!Number.isSafeInteger(lane.value) || lane.value <= 0) {
+ throw new Error('Agent trigger lane publisher has an invalid sequence');
+ }
+ const publisherFence = {
+ _id: lane._id,
+ value: lane.value,
+ publisherDeliveryId,
+ };
+ if (lane.value === 1) {
+ const deleted = await LaneSequence().deleteOne(publisherFence);
+ return deleted.deletedCount === 1;
+ }
+ const released = await LaneSequence().updateOne(publisherFence, {
+ $inc: { value: -1 },
+ $set: { cleanupRequestedAt: new Date() },
+ $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },
+ });
+ return released.modifiedCount === 1;
+ }
+
+ async function recoverLanePublisher(lane: IAgentTriggerLaneSequence): Promise<boolean> {
+ const publisherDeliveryId = lane.publisherDeliveryId;
+ if (publisherDeliveryId == null) {
+ return false;
+ }
+ if (!Number.isSafeInteger(lane.value) || lane.value <= 0) {
+ throw new Error('Agent trigger lane publisher has an invalid sequence');
+ }
+
+ const published = await Delivery().updateOne(
+ { _id: publisherDeliveryId, orderingKey: lane._id, status: 'staging' },
+ {
+ $set: { status: 'pending', laneSequence: lane.value },
+ $unset: { stagingRecoveryAt: 1 },
+ },
+ );
+ let publicationCommitted = published.modifiedCount === 1;
+ if (published.modifiedCount === 0) {
+ const current = await Delivery()
+ .findById(publisherDeliveryId)
+ .select('orderingKey laneSequence status')
+ .lean<Pick<IAgentTriggerDelivery, 'orderingKey' | 'laneSequence' | 'status'>>();
+ publicationCommitted =
+ current != null &&
+ current.orderingKey === lane._id &&
+ current.status !== 'staging' &&
+ current.laneSequence === lane.value;
+ if (current?.orderingKey === lane._id && current.status === 'staging') {
+ throw new Error('Failed to publish the reserved agent trigger delivery');
+ }
+ }
+
+ const publisherFence = {
+ _id: lane._id,
+ value: lane.value,
+ publisherDeliveryId,
+ };
+ if (publicationCommitted) {
+ const released = await LaneSequence().updateOne(publisherFence, {
+ $set: { tailDeliveryId: publisherDeliveryId },
+ $unset: { publisherDeliveryId: 1, publisherStartedAt: 1 },
+ });
+ return published.modifiedCount === 1 || released.modifiedCount === 1;
+ }
+
+ // The selected row was published by a competing replica before this
+ // reservation was acquired (or was removed by account cleanup). No later
+ // reservation can exist while this fence is held, so the unused sequence
+ // can be rolled back without creating a gap or regressing the lane tail.
+ return abandonLanePublisher(lane);
+ }
+
+ async function publishStagedDelivery(
+ delivery: IAgentTriggerDelivery,
+ ): Promise<IAgentTriggerDelivery> {
+ if (delivery._id == null) {
+ throw new Error('Staged agent trigger delivery is missing its identity');
+ }
+ const deliveryId = delivery._id;
+ const orderingKey = delivery.orderingKey;
+
+ for (;;) {
+ const current = await Delivery().findById(deliveryId).lean<IAgentTriggerDelivery>();
+ if (current == null) {
+ throw new Error('Staged agent trigger delivery disappeared before publication');
+ }
+ if (current.status !== 'staging') {
+ return current;
+ }
+ if ((await UserPurge().exists({ _id: current.user })) != null) {
+ return current;
+ }
+
+ const lane = await LaneSequence().findById(orderingKey).lean<IAgentTriggerLaneSequence>();
+ if (lane?.publisherDeliveryId != null) {
+ if ((await UserPurge().exists({ _id: lane.user })) != null) {
+ return current;
+ }
+ await recoverLanePublisher(lane);
+ continue;
+ }
+
+ // The durable staging row exists before sequence allocation. Publishing
+ // the oldest visible row first lets any replica repair a writer that
+ // stopped in that gap without allowing a later enqueue to overtake it.
+ const next = await Delivery()
+ .findOne({ orderingKey, status: 'staging' })
+ // updatedAt is the current staging admission time: initial enqueue
+ // sets it on insert, while explicit requeue refreshes it so an old
+ // dead letter is admitted behind staging work that already exists.
+ .sort({ updatedAt: 1, _id: 1 })
+ .lean<IAgentTriggerDelivery>();
+ if (next?._id == null) {
+ continue;
+ }
+
+ let acquired: IAgentTriggerLaneSequence | null = null;
+ try {
+ acquired = await LaneSequence()
+ .findOneAndUpdate(
+ { _id: orderingKey, publisherDeliveryId: { $exists: false } },
+ {
+ $inc: { value: 1 },
+ $set: {
+ publisherDeliveryId: next._id,
+ publisherStartedAt: new Date(),
+ },
+ $unset: { cleanupRequestedAt: 1 },
+ $setOnInsert: {
+ user: next.user,
+ ...(next.tenantId != null && { tenantId: next.tenantId }),
+ },
+ },
+ { new: true, upsert: true, setDefaultsOnInsert: true },
+ )
+ .lean<IAgentTriggerLaneSequence>();
+ } catch (error) {
+ if ((error as DuplicateKeyError)?.code !== DUPLICATE_KEY) {
+ throw error;
+ }
+ }
+ if (acquired == null) {
+ continue;
+ }
+
+ // Account deletion creates this durable marker before it can delete
+ // rows or lanes. Recheck after acquisition to close the cross-collection
+ // race in which the marker appears between the first check and upsert.
+ if ((await UserPurge().exists({ _id: next.user })) != null) {
+ await abandonLanePublisher(acquired);
+ return current;
+ }
+
+ await recoverLanePublisher(acquired);
+ }
+ }
+
+ async function enqueueAgentTriggerDelivery(
+ input: EnqueueAgentTriggerDeliveryInput,
+ ): Promise<{ delivery: AgentTriggerDeliveryRecord; replayed: boolean }> {
+ let staged: IAgentTriggerDelivery;
+ let replayed = false;
+ try {
+ const created = await Delivery().create({
+ ...input,
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ stagingRecoveryAt: new Date(),
+ });
+ staged = created.toObject();
+ } catch (error) {
+ if ((error as DuplicateKeyError)?.code !== DUPLICATE_KEY) {
+ throw error;
+ }
+ const existing = await Delivery()
+ .findOne({ deliveryKey: input.deliveryKey })
+ .lean<IAgentTriggerDelivery>();
+ if (existing == null) {
+ throw error;
+ }
+ if (existing.fingerprint !== input.fingerprint) {
+ throw new AgentTriggerDeliveryConflictError(input.deliveryKey);
+ }
+ if (existing.status !== 'staging') {
+ return { delivery: toRecord(existing), replayed: true };
+ }
+ staged = existing;
+ replayed = true;
+ }
+
+ const published = await publishStagedDelivery(staged);
+ return { delivery: toRecord(published), replayed };
+ }
+
+ /** Repairs abandoned reservations and staging rows left by crashed writers. */
+ async function recoverAgentTriggerLanePublications(
+ limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ ): Promise<number> {
+ if (!Number.isSafeInteger(limit) || limit <= 0) {
+ throw new TypeError('Agent trigger lane recovery limit must be a positive integer');
+ }
+ const boundedLimit = Math.min(limit, MAX_PURGE_RECOVERY_LIMIT);
+ const lanes = await LaneSequence()
+ .find({
+ publisherDeliveryId: { $exists: true },
+ publisherStartedAt: { $exists: true },
+ })
+ .sort({ publisherStartedAt: 1, _id: 1 })
+ .limit(boundedLimit)
+ .lean<IAgentTriggerLaneSequence[]>();
+ let recovered = 0;
+ const recoveryCursor = new Date();
+ for (const lane of lanes) {
+ if ((await UserPurge().exists({ _id: lane.user })) != null) {
+ if (lane.publisherDeliveryId != null && lane.publisherStartedAt != null) {
+ await LaneSequence().updateOne(
+ {
+ _id: lane._id,
+ value: lane.value,
+ publisherDeliveryId: lane.publisherDeliveryId,
+ publisherStartedAt: lane.publisherStartedAt,
+ },
+ { $set: { publisherStartedAt: recoveryCursor } },
+ { timestamps: false },
+ );
+ }
+ continue;
+ }
+ if (await recoverLanePublisher(lane)) {
+ recovered += 1;
+ }
+ }
+
+ let remaining = boundedLimit - recovered;
+ if (remaining <= 0) {
+ return recovered;
+ }
+ /** Pre-cursor replicas can leave staging rows outside the sparse recovery
+ * index. Backfill them in bounded batches before using the indexed scan. */
+ const legacyStaged = await Delivery()
+ .find({ status: 'staging', stagingRecoveryAt: { $exists: false } })
+ .sort({ updatedAt: 1, _id: 1 })
+ .limit(remaining)
+ .lean<IAgentTriggerDelivery[]>();
+ const legacyCursor = new Date();
+ for (const delivery of legacyStaged) {
+ if (delivery._id != null) {
+ await Delivery().updateOne(
+ { _id: delivery._id, status: 'staging', stagingRecoveryAt: { $exists: false } },
+ { $set: { stagingRecoveryAt: legacyCursor } },
+ { timestamps: false },
+ );
+ }
+ }
+ remaining -= legacyStaged.length;
+ const legacyIds = legacyStaged.flatMap((delivery) =>
+ delivery._id != null ? [delivery._id] : [],
+ );
+ const indexedStaged =
+ remaining > 0
+ ? await Delivery()
+ .find({
+ status: 'staging',
+ stagingRecoveryAt: { $exists: true },
+ ...(legacyIds.length > 0 && { _id: { $nin: legacyIds } }),
+ })
+ .sort({ stagingRecoveryAt: 1, _id: 1 })
+ .limit(remaining)
+ .lean<IAgentTriggerDelivery[]>()
+ : [];
+ const staged = [
+ ...legacyStaged.map((delivery) => ({ ...delivery, stagingRecoveryAt: legacyCursor })),
+ ...indexedStaged,
+ ];
+ const stagingRecoveryCursor = new Date();
+ const rotateStagingRecovery = async (delivery: IAgentTriggerDelivery): Promise<void> => {
+ if (delivery._id == null || delivery.stagingRecoveryAt == null) {
+ return;
+ }
+ await Delivery().updateOne(
+ {
+ _id: delivery._id,
+ status: 'staging',
+ stagingRecoveryAt: delivery.stagingRecoveryAt,
+ },
+ { $set: { stagingRecoveryAt: stagingRecoveryCursor } },
+ { timestamps: false },
+ );
+ };
+ for (const delivery of staged) {
+ try {
+ const published = await publishStagedDelivery(delivery);
+ if (published.status !== 'staging') {
+ recovered += 1;
+ } else {
+ await rotateStagingRecovery(delivery);
+ }
+ } catch (error) {
+ // Account deletion can remove a staged row while a replica is helping
+ // it. Suppress only that confirmed race; operational failures remain
+ // visible to the maintenance loop.
+ if (delivery._id != null && (await Delivery().exists({ _id: delivery._id })) == null) {
+ continue;
+ }
+ await rotateStagingRecovery(delivery);
+ throw error;
+ }
+ }
+ return recovered;
+ }
+
+ async function reclaimLaneIfInactive(orderingKey: string): Promise<boolean> {
+ const lane = await LaneSequence().findById(orderingKey).lean<IAgentTriggerLaneSequence>();
+ if (lane?.cleanupRequestedAt == null) {
+ return false;
+ }
+ const stillRetained = await Delivery().exists({
+ orderingKey,
+ status: { $in: ['staging', 'pending', 'leased', 'dead'] },
+ });
+ if (stillRetained != null) {
+ await LaneSequence().updateOne(
+ { _id: orderingKey, cleanupRequestedAt: lane.cleanupRequestedAt },
+ { $unset: { cleanupRequestedAt: 1 } },
+ );
+ return false;
+ }
+ const deleted = await LaneSequence().deleteOne({
+ _id: orderingKey,
+ ...(lane.tailDeliveryId == null
+ ? { tailDeliveryId: { $exists: false } }
+ : { tailDeliveryId: lane.tailDeliveryId }),
+ cleanupRequestedAt: lane.cleanupRequestedAt,
+ publisherDeliveryId: { $exists: false },
+ });
+ return deleted.deletedCount === 1;
+ }
+
+ async function fulfillLaneCleanupRequest(
+ delivery: Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>,
+ ): Promise<boolean> {
+ if (delivery._id == null || delivery.laneCleanupPendingAt == null) {
+ return false;
+ }
+ await LaneSequence().updateOne(
+ { _id: delivery.orderingKey },
+ { $set: { cleanupRequestedAt: delivery.laneCleanupPendingAt } },
+ );
+ await Delivery().updateOne(
+ {
+ _id: delivery._id,
+ status: 'succeeded',
+ laneCleanupPendingAt: delivery.laneCleanupPendingAt,
+ },
+ { $unset: { laneCleanupPendingAt: 1 } },
+ { timestamps: false },
+ );
+ return reclaimLaneIfInactive(delivery.orderingKey);
+ }
+
+ /** Bounds high-cardinality ordering metadata after the final retained job settles. */
+ async function reclaimInactiveAgentTriggerLanes(
+ limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ ): Promise<number> {
+ if (!Number.isSafeInteger(limit) || limit <= 0) {
+ throw new TypeError('Agent trigger lane reclamation limit must be a positive integer');
+ }
+ const boundedLimit = Math.min(limit, MAX_PURGE_RECOVERY_LIMIT);
+ const pendingCleanup = await Delivery()
+ .find({ status: 'succeeded', laneCleanupPendingAt: { $exists: true } })
+ .sort({ laneCleanupPendingAt: 1, _id: 1 })
+ .limit(boundedLimit)
+ .select('_id orderingKey laneCleanupPendingAt')
+ .lean<Array<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>>();
+ let reclaimed = 0;
+ for (const delivery of pendingCleanup) {
+ if (await fulfillLaneCleanupRequest(delivery)) {
+ reclaimed += 1;
+ }
+ }
+
+ const remaining = boundedLimit - pendingCleanup.length;
+ if (remaining <= 0) {
+ return reclaimed;
+ }
+ const lanes = await LaneSequence()
+ .find({ cleanupRequestedAt: { $exists: true } })
+ .sort({ cleanupRequestedAt: 1, _id: 1 })
+ .limit(remaining)
+ .select('_id')
+ .lean<Array<Pick<IAgentTriggerLaneSequence, '_id'>>>();
+ for (const lane of lanes) {
+ if (await reclaimLaneIfInactive(lane._id)) {
+ reclaimed += 1;
+ }
+ }
+ return reclaimed;
+ }
+
+ async function claimNextAgentTriggerDelivery(input: {
+ workerId: string;
+ claimToken: string;
+ now: Date;
+ leaseUntil: Date;
+ }): Promise<AgentTriggerDeliveryClaim | null> {
+ const expiredBefore = new Date(input.now.getTime() - LEASE_SKEW_MARGIN_MS);
+ const claimed = await Delivery()
+ .findOneAndUpdate(
+ {
+ $or: [
+ { status: 'pending', availableAt: { $lte: input.now } },
+ { status: 'leased', leaseUntil: { $lte: expiredBefore } },
+ ],
+ },
+ {
+ $set: {
+ status: 'leased',
+ leaseBy: input.workerId,
+ leaseUntil: input.leaseUntil,
+ claimToken: input.claimToken,
+ },
+ $unset: { settledAt: 1, expiresAt: 1 },
+ },
+ { new: true, sort: { availableAt: 1, createdAt: 1, _id: 1 } },
+ )
+ .lean<IAgentTriggerDelivery>();
+ return requireClaim(claimed);
+ }
+
+ async function findEarlierAgentTriggerDelivery(
+ delivery: Pick<AgentTriggerDeliveryRecord, 'orderingKey' | 'laneSequence'>,
+ ): Promise<AgentTriggerOrderingBlock | null> {
+ const earlier = await Delivery()
+ .findOne({
+ orderingKey: delivery.orderingKey,
+ status: { $in: ['pending', 'leased'] },
+ laneSequence: { $lt: delivery.laneSequence },
+ })
+ .sort({ laneSequence: 1 })
+ .select('availableAt leaseUntil')
+ .lean<Pick<IAgentTriggerDelivery, 'availableAt' | 'leaseUntil'>>();
+ if (earlier == null) {
+ return null;
+ }
+ return {
+ availableAt: earlier.availableAt,
+ ...(earlier.leaseUntil != null && { leaseUntil: earlier.leaseUntil }),
+ };
+ }
+
+ const fence = (input: AgentTriggerDeliveryFence) => ({
+ _id: input.id,
+ status: 'leased',
+ leaseBy: input.workerId,
+ claimToken: input.claimToken,
+ });
+
+ async function releaseAgentTriggerDelivery(
+ input: AgentTriggerDeliveryFence & { availableAt: Date },
+ ): Promise<boolean> {
+ const result = await Delivery().updateOne(fence(input), {
+ $set: { status: 'pending', availableAt: input.availableAt },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
+ });
+ return result.modifiedCount === 1;
+ }
+
+ async function beginAgentTriggerDeliveryAttempt(
+ input: AgentTriggerDeliveryFence & { now: Date },
+ ): Promise<number | null> {
+ const updated = await Delivery()
+ .findOneAndUpdate(
+ { ...fence(input), leaseUntil: { $gt: input.now } },
+ { $inc: { attempts: 1 } },
+ { new: true },
+ )
+ .select('attempts')
+ .lean<Pick<IAgentTriggerDelivery, 'attempts'>>();
+ return updated?.attempts ?? null;
+ }
+
+ /** Releases a pre-dispatch deferral and restores the attempt consumed by beginAttempt. */
+ async function deferAgentTriggerDeliveryAttempt(
+ input: AgentTriggerDeliveryFence & { attempt: number; availableAt: Date },
+ ): Promise<boolean> {
+ if (!Number.isSafeInteger(input.attempt) || input.attempt <= 0) {
+ throw new TypeError('attempt must be a positive integer');
+ }
+ const result = await Delivery().updateOne(
+ { ...fence(input), attempts: input.attempt },
+ {
+ $inc: { attempts: -1 },
+ $set: { status: 'pending', availableAt: input.availableAt },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1 },
+ },
+ );
+ return result.modifiedCount === 1;
+ }
+
+ async function completeAgentTriggerDelivery(
+ input: AgentTriggerDeliveryFence & {
+ attempt: number;
+ result: unknown;
+ settledAt: Date;
+ },
+ ): Promise<boolean> {
+ const completed = await Delivery()
+ .findOneAndUpdate(
+ fence(input),
+ {
+ $set: {
+ status: 'succeeded',
+ result: input.result,
+ settledAt: input.settledAt,
+ expiresAt: new Date(input.settledAt.getTime() + SUCCESS_RETENTION_MS),
+ laneCleanupPendingAt: input.settledAt,
+ },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, lastError: 1 },
+ $push: {
+ history: {
+ $each: [
+ {
+ attempt: input.attempt,
+ outcome: 'succeeded',
+ at: input.settledAt,
+ workerId: input.workerId,
+ },
+ ],
+ $slice: -HISTORY_LIMIT,
+ },
+ },
+ },
+ { new: true },
+ )
+ .select('_id orderingKey laneCleanupPendingAt')
+ .lean<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>();
+ if (completed?._id == null) {
+ return false;
+ }
+
+ try {
+ await fulfillLaneCleanupRequest(completed);
+ } catch (error) {
+ // Delivery success is authoritative; counter reclamation is safe to retry later.
+ logger.warn('[agent-triggers] failed to reclaim an inactive ordering lane', {
+ orderingKey: completed.orderingKey,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ return true;
+ }
+
+ async function retryAgentTriggerDelivery(
+ input: AgentTriggerDeliveryFence & {
+ attempt: number;
+ error: AgentTriggerDeliveryFailure;
+ availableAt: Date;
+ },
+ ): Promise<boolean> {
+ const error = normalizeFailure(input.error);
+ const result = await Delivery().updateOne(fence(input), {
+ $set: { status: 'pending', availableAt: input.availableAt, lastError: error },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, settledAt: 1, expiresAt: 1 },
+ $push: {
+ history: {
+ $each: [
+ {
+ attempt: input.attempt,
+ outcome: 'retry',
+ at: error.attemptedAt,
+ workerId: input.workerId,
+ error,
+ },
+ ],
+ $slice: -HISTORY_LIMIT,
+ },
+ },
+ });
+ return result.modifiedCount === 1;
+ }
+
+ async function deadLetterAgentTriggerDelivery(
+ input: AgentTriggerDeliveryFence & {
+ attempt: number;
+ error: AgentTriggerDeliveryFailure;
+ settledAt: Date;
+ },
+ ): Promise<boolean> {
+ const error = normalizeFailure(input.error);
+ const result = await Delivery().updateOne(fence(input), {
+ $set: { status: 'dead', settledAt: input.settledAt, lastError: error },
+ $unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },
+ $push: {
+ history: {
+ $each: [
+ {
+ attempt: input.attempt,
+ outcome: 'dead',
+ at: input.settledAt,
+ workerId: input.workerId,
+ error,
+ },
+ ],
+ $slice: -HISTORY_LIMIT,
+ },
+ },
+ });
+ return result.modifiedCount === 1;
+ }
+
+ async function getAgentTriggerDelivery(
+ deliveryKey: string,
+ ): Promise<AgentTriggerDeliveryRecord | null> {
+ const delivery = await Delivery().findOne({ deliveryKey }).lean<IAgentTriggerDelivery>();
+ return delivery == null ? null : toRecord(delivery);
+ }
+
+ async function getAgentTriggerDeadLetters(limit = 50): Promise<AgentTriggerDeliveryRecord[]> {
+ if (!Number.isSafeInteger(limit) || limit <= 0) {
+ throw new TypeError('Agent trigger dead-letter limit must be a positive integer');
+ }
+ const boundedLimit = Math.min(limit, MAX_DEAD_LETTER_LIMIT);
+ const deliveries = await Delivery()
+ .find({ status: 'dead' })
+ .sort({ updatedAt: -1, _id: -1 })
+ .limit(boundedLimit)
+ .lean<IAgentTriggerDelivery[]>();
+ return deliveries.map(toRecord);
+ }
+
+ async function requeueAgentTriggerDelivery(
+ id: string,
+ availableAt: Date,
+ ): Promise<AgentTriggerDeliveryRecord | null> {
+ const staged = await Delivery()
+ .findOneAndUpdate(
+ { _id: id, status: 'dead' },
+ {
+ $set: {
+ status: 'staging',
+ laneSequence: 0,
+ attempts: 0,
+ availableAt,
+ stagingRecoveryAt: new Date(),
+ },
+ $unset: {
+ leaseBy: 1,
+ leaseUntil: 1,
+ claimToken: 1,
+ lastError: 1,
+ result: 1,
+ settledAt: 1,
+ expiresAt: 1,
+ laneCleanupPendingAt: 1,
+ },
+ $inc: { requeueCount: 1 },
+ },
+ { new: true },
+ )
+ .lean<IAgentTriggerDelivery>();
+ if (staged == null) {
+ return null;
+ }
+
+ // Requeue is a new lane admission. Retaining the dead letter's original
+ // sequence could let it overlap a newer delivery that is already running.
+ return toRecord(await publishStagedDelivery(staged));
+ }
+
+ async function countActiveAgentTriggerDeliveriesByUser(
+ user: string | Types.ObjectId,
+ now: Date,
+ ): Promise<number> {
+ return Delivery().countDocuments({ user, status: 'leased', leaseUntil: { $gt: now } });
+ }
+
+ function requireValidFence(fenceStartedAt: Date): void {
+ if (!(fenceStartedAt instanceof Date) || !Number.isFinite(fenceStartedAt.getTime())) {
+ throw new TypeError('fenceStartedAt must be a valid Date');
+ }
+ }
+
+ /** Arms independently retryable cleanup before the user deletion can commit. */
+ async function prepareAgentTriggerUserPurge(
+ user: string | Types.ObjectId,
+ fenceStartedAt: Date,
+ tenantId?: string,
+ ): Promise<void> {
+ requireValidFence(fenceStartedAt);
+ const ownsFence = await mongoose.models.User.exists({
+ _id: user,
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ if (ownsFence == null) {
+ throw new Error('Cannot prepare trigger purge without owning the user deletion fence');
+ }
+ await UserPurge().updateOne(
+ { _id: user },
+ {
+ $set: {
+ fenceStartedAt,
+ ...(tenantId != null && { tenantId }),
+ },
+ },
+ { upsert: true },
+ );
+ }
+
+ /** Disarms only the pre-commit deletion attempt that owns this marker. */
+ async function cancelAgentTriggerUserPurge(
+ user: string | Types.ObjectId,
+ fenceStartedAt: Date,
+ ): Promise<boolean> {
+ requireValidFence(fenceStartedAt);
+ const ownsLiveFence = await mongoose.models.User.exists({
+ _id: user,
+ agentTriggerDeletionStartedAt: fenceStartedAt,
+ });
+ if (ownsLiveFence == null) {
+ return false;
+ }
+ const result = await UserPurge().deleteOne({ _id: user, fenceStartedAt });
+ return result.deletedCount === 1;
+ }
+
+ /** Recovers cleanup markers whose users are gone; active-user markers are never destructive. */
+ async function recoverAgentTriggerUserPurges(
+ limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ ): Promise<number> {
+ if (!Number.isSafeInteger(limit) || limit <= 0) {
+ throw new TypeError('Agent trigger purge recovery limit must be a positive integer');
+ }
+ const markers = await UserPurge()
+ .find({})
+ .sort({ updatedAt: 1, _id: 1 })
+ .limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
+ .lean<IAgentTriggerUserPurge[]>();
+ let recovered = 0;
+ for (const marker of markers) {
+ const user = await mongoose.models.User.findById(marker._id)
+ .select('agentTriggerDeletionStartedAt')
+ .lean<{ agentTriggerDeletionStartedAt?: Date }>();
+ if (user != null) {
+ if (user.agentTriggerDeletionStartedAt?.getTime() === marker.fenceStartedAt.getTime()) {
+ // Move a still-owned pre-commit marker behind this bounded scan so
+ // it cannot starve later markers whose users are already gone.
+ await UserPurge().updateOne(
+ { _id: marker._id, fenceStartedAt: marker.fenceStartedAt },
+ { $currentDate: { updatedAt: true } },
+ );
+ continue;
+ }
+ await UserPurge().deleteOne({ _id: marker._id, fenceStartedAt: marker.fenceStartedAt });
+ continue;
+ }
+ await Promise.all([
+ Delivery().deleteMany({ user: marker._id }),
+ LaneSequence().deleteMany({ user: marker._id }),
+ ]);
+ const result = await UserPurge().deleteOne({
+ _id: marker._id,
+ fenceStartedAt: marker.fenceStartedAt,
+ });
+ recovered += result.deletedCount;
+ }
+ return recovered;
+ }
+
+ async function deleteAgentTriggerDeliveriesByUser(user: string | Types.ObjectId): Promise<void> {
+ await Promise.all([Delivery().deleteMany({ user }), LaneSequence().deleteMany({ user })]);
+ await UserPurge().deleteOne({ _id: user });
+ }
+
+ return {
+ ensureAgentTriggerDeliveryIndexes,
+ enqueueAgentTriggerDelivery,
+ claimNextAgentTriggerDelivery,
+ findEarlierAgentTriggerDelivery,
+ releaseAgentTriggerDelivery,
+ beginAgentTriggerDeliveryAttempt,
+ deferAgentTriggerDeliveryAttempt,
+ completeAgentTriggerDelivery,
+ retryAgentTriggerDelivery,
+ deadLetterAgentTriggerDelivery,
+ getAgentTriggerDelivery,
+ getAgentTriggerDeadLetters,
+ requeueAgentTriggerDelivery,
+ countActiveAgentTriggerDeliveriesByUser,
+ recoverAgentTriggerLanePublications,
+ reclaimInactiveAgentTriggerLanes,
+ prepareAgentTriggerUserPurge,
+ cancelAgentTriggerUserPurge,
+ recoverAgentTriggerUserPurges,
+ deleteAgentTriggerDeliveriesByUser,
+ };
+}
diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts
index 1270a03c5b..9540026232 100644
--- a/packages/data-schemas/src/methods/user.methods.spec.ts
+++ b/packages/data-schemas/src/methods/user.methods.spec.ts
@@ -2,8 +2,8 @@ import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
import type * as t from '~/types';
+import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';
import balanceSchema from '~/schema/balance';
-import { createUserMethods } from './user';
import userSchema from '~/schema/user';
/** Mocking crypto for generateToken */
@@ -628,6 +628,85 @@ describe('User Methods - Database Tests', () => {
});
});
+ describe('agent trigger account-deletion fence', () => {
+ test('blocks trigger admission until the owning deletion attempt releases it', async () => {
+ const user = await User.create({
+ name: 'Trigger Fence',
+ email: '[email protected]',
+ provider: 'local',
+ });
+ const userId = user._id.toString();
+ const startedAt = new Date('2026-08-17T12:00:00.000Z');
+
+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);
+ await expect(methods.beginAgentTriggerUserDeletion(userId, startedAt)).resolves.toBe(
+ 'acquired',
+ );
+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);
+ await expect(
+ methods.beginAgentTriggerUserDeletion(userId, new Date(startedAt.getTime() + 1)),
+ ).resolves.toBe('in_progress');
+ await expect(
+ methods.cancelAgentTriggerUserDeletion(userId, new Date(startedAt.getTime() + 1)),
+ ).resolves.toBe(false);
+ await expect(methods.cancelAgentTriggerUserDeletion(userId, startedAt)).resolves.toBe(true);
+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);
+ });
+
+ test('reports a missing principal without creating a deletion fence', async () => {
+ const userId = new mongoose.Types.ObjectId().toString();
+
+ await expect(methods.beginAgentTriggerUserDeletion(userId, new Date())).resolves.toBe(
+ 'missing',
+ );
+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);
+ });
+
+ test('requires explicit stale-fence recovery and preserves successor ownership', async () => {
+ const user = await User.create({
+ name: 'Stale Trigger Fence',
+ email: '[email protected]',
+ provider: 'local',
+ });
+ const userId = user._id.toString();
+ const abandonedAt = new Date('2026-08-17T12:00:00.000Z');
+ const takeoverAt = new Date(abandonedAt.getTime() + USER_DELETION_FENCE_STALE_MS + 1);
+
+ await expect(methods.beginAgentTriggerUserDeletion(userId, abandonedAt)).resolves.toBe(
+ 'acquired',
+ );
+ await expect(methods.beginAgentTriggerUserDeletion(userId, takeoverAt)).resolves.toBe(
+ 'in_progress',
+ );
+ await expect(
+ methods.recoverStaleAgentTriggerUserDeletion(
+ userId,
+ new Date(abandonedAt.getTime() + USER_DELETION_FENCE_STALE_MS - 1),
+ ),
+ ).resolves.toBe('in_progress');
+ await expect(methods.recoverStaleAgentTriggerUserDeletion(userId, takeoverAt)).resolves.toBe(
+ 'acquired',
+ );
+ await expect(methods.cancelAgentTriggerUserDeletion(userId, abandonedAt)).resolves.toBe(
+ false,
+ );
+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(false);
+ await expect(methods.cancelAgentTriggerUserDeletion(userId, takeoverAt)).resolves.toBe(true);
+ await expect(methods.isAgentTriggerPrincipalActive(userId)).resolves.toBe(true);
+ });
+
+ test('rejects invalid deletion-fence timestamps', async () => {
+ const userId = new mongoose.Types.ObjectId().toString();
+
+ await expect(
+ methods.beginAgentTriggerUserDeletion(userId, new Date(Number.NaN)),
+ ).rejects.toThrow('startedAt must be a valid Date');
+ await expect(
+ methods.recoverStaleAgentTriggerUserDeletion(userId, new Date(Number.NaN)),
+ ).rejects.toThrow('recoveredAt must be a valid Date');
+ });
+ });
+
describe('countUsers', () => {
test('should count all users', async () => {
await User.create([
diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts
index d03eb7fed6..2d324bfcdc 100644
--- a/packages/data-schemas/src/methods/user.ts
+++ b/packages/data-schemas/src/methods/user.ts
@@ -12,6 +12,8 @@ import { signPayload } from '~/crypto';
/** Default JWT session expiry: 15 minutes in milliseconds */
export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15;
+/** Minimum age before an explicitly offline operator may recover an abandoned deletion fence. */
+export const USER_DELETION_FENCE_STALE_MS: number = 15 * 60_000;
interface UserMethodDeps {
getCache?: (key: string) => CacheStore | undefined;
@@ -117,6 +119,16 @@ export function createUserMethods(
>;
getUserById: (userId: string, fieldsToSelect?: string | string[] | null) => Promise<IUser | null>;
generateToken: (user: IUser, expiresIn?: number) => Promise<string>;
+ beginAgentTriggerUserDeletion: (
+ userId: string,
+ startedAt: Date,
+ ) => Promise<'acquired' | 'in_progress' | 'missing'>;
+ recoverStaleAgentTriggerUserDeletion: (
+ userId: string,
+ recoveredAt: Date,
+ ) => Promise<'acquired' | 'in_progress' | 'missing'>;
+ cancelAgentTriggerUserDeletion: (userId: string, startedAt: Date) => Promise<boolean>;
+ isAgentTriggerPrincipalActive: (userId: string) => Promise<boolean>;
deleteUserById: (userId: string) => Promise<UserDeleteResult>;
updateUserPlugins: (
userId: string,
@@ -376,6 +388,70 @@ export function createUserMethods(
}
}
+ /** Establishes the durable admission fence used while account deletion drains triggers. */
+ async function beginAgentTriggerUserDeletion(
+ userId: string,
+ startedAt: Date,
+ ): Promise<'acquired' | 'in_progress' | 'missing'> {
+ if (!(startedAt instanceof Date) || !Number.isFinite(startedAt.getTime())) {
+ throw new TypeError('startedAt must be a valid Date');
+ }
+ const User = mongoose.models.User;
+ const result = await User.updateOne(
+ { _id: userId, agentTriggerDeletionStartedAt: { $exists: false } },
+ { $set: { agentTriggerDeletionStartedAt: startedAt } },
+ );
+ if (result.modifiedCount === 1) {
+ await invalidateAuthUserDocCache(userId);
+ return 'acquired';
+ }
+ return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress';
+ }
+
+ /** Replaces an abandoned fence only for an operator-confirmed offline deployment.
+ * Normal request paths must never call this: age alone cannot prove the prior owner died. */
+ async function recoverStaleAgentTriggerUserDeletion(
+ userId: string,
+ recoveredAt: Date,
+ ): Promise<'acquired' | 'in_progress' | 'missing'> {
+ if (!(recoveredAt instanceof Date) || !Number.isFinite(recoveredAt.getTime())) {
+ throw new TypeError('recoveredAt must be a valid Date');
+ }
+ const User = mongoose.models.User;
+ const staleBefore = new Date(recoveredAt.getTime() - USER_DELETION_FENCE_STALE_MS);
+ const result = await User.updateOne(
+ { _id: userId, agentTriggerDeletionStartedAt: { $lte: staleBefore } },
+ { $set: { agentTriggerDeletionStartedAt: recoveredAt } },
+ );
+ if (result.modifiedCount === 1) {
+ await invalidateAuthUserDocCache(userId);
+ return 'acquired';
+ }
+ return (await User.exists({ _id: userId })) == null ? 'missing' : 'in_progress';
+ }
+
+ /** Releases only the account-deletion attempt that owns this exact fence. */
+ async function cancelAgentTriggerUserDeletion(userId: string, startedAt: Date): Promise<boolean> {
+ const User = mongoose.models.User;
+ const result = await User.updateOne(
+ { _id: userId, agentTriggerDeletionStartedAt: startedAt },
+ { $unset: { agentTriggerDeletionStartedAt: 1 } },
+ );
+ if (result.modifiedCount === 1) {
+ await invalidateAuthUserDocCache(userId);
+ return true;
+ }
+ return false;
+ }
+
+ async function isAgentTriggerPrincipalActive(userId: string): Promise<boolean> {
+ const User = mongoose.models.User;
+ return (
+ (await User.exists({ _id: userId, agentTriggerDeletionStartedAt: { $exists: false } })) !=
+ null
+ );
+ }
+
/**
* Generates a JWT token for a given user.
* @param user - The user object
@@ -627,6 +703,10 @@ export function createUserMethods(
searchUsers,
getUserById,
generateToken,
+ beginAgentTriggerUserDeletion,
+ recoverStaleAgentTriggerUserDeletion,
+ cancelAgentTriggerUserDeletion,
+ isAgentTriggerPrincipalActive,
deleteUserById,
updateUserPlugins,
toggleUserMemories,
diff --git a/packages/data-schemas/src/models/index.ts b/packages/data-schemas/src/models/index.ts
index ed1e56d9a5..06d6a818f8 100644
--- a/packages/data-schemas/src/models/index.ts
+++ b/packages/data-schemas/src/models/index.ts
@@ -1,4 +1,7 @@
+import { createAgentTriggerLaneSequenceModel } from './triggerLaneSequence';
import { createSkillSyncCredentialModel } from './skillSyncCredential';
+import { createAgentTriggerUserPurgeModel } from './triggerUserPurge';
+import { createAgentTriggerDeliveryModel } from './triggerDelivery';
import { createSkillSyncStatusModel } from './skillSyncStatus';
import { createConversationTagModel } from './conversationTag';
import { createAgentCategoryModel } from './agentCategory';
@@ -78,6 +81,9 @@ export function createModels(mongoose: typeof import('mongoose')): {
AuditLog: ReturnType<typeof createAuditLogModel>;
Group: ReturnType<typeof createGroupModel>;
Config: ReturnType<typeof createConfigModel>;
+ AgentTriggerDelivery: ReturnType<typeof createAgentTriggerDeliveryModel>;
+ AgentTriggerLaneSequence: ReturnType<typeof createAgentTriggerLaneSequenceModel>;
+ AgentTriggerUserPurge: ReturnType<typeof createAgentTriggerUserPurgeModel>;
} {
const models = {
User: createUserModel(mongoose),
@@ -117,6 +123,9 @@ export function createModels(mongoose: typeof import('mongoose')): {
AuditLog: createAuditLogModel(mongoose),
Group: createGroupModel(mongoose),
Config: createConfigModel(mongoose),
+ AgentTriggerDelivery: createAgentTriggerDeliveryModel(mongoose),
+ AgentTriggerLaneSequence: createAgentTriggerLaneSequenceModel(mongoose),
+ AgentTriggerUserPurge: createAgentTriggerUserPurgeModel(mongoose),
};
/**
* Background index builds fail silently unless an 'index' listener is
diff --git a/packages/data-schemas/src/models/triggerDelivery.ts b/packages/data-schemas/src/models/triggerDelivery.ts
new file mode 100644
index 0000000000..0802429465
--- /dev/null
+++ b/packages/data-schemas/src/models/triggerDelivery.ts
@@ -0,0 +1,14 @@
+import { Model } from 'mongoose';
+import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';
+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';
+import triggerDeliverySchema from '~/schema/triggerDelivery';
+
+export function createAgentTriggerDeliveryModel(
+ mongoose: typeof import('mongoose'),
+): Model<IAgentTriggerDeliveryDocument> {
+ applyTenantIsolation(triggerDeliverySchema);
+ return (
+ mongoose.models.AgentTriggerDelivery ||
+ mongoose.model<IAgentTriggerDeliveryDocument>('AgentTriggerDelivery', triggerDeliverySchema)
+ );
+}
diff --git a/packages/data-schemas/src/models/triggerLaneSequence.ts b/packages/data-schemas/src/models/triggerLaneSequence.ts
new file mode 100644
index 0000000000..04e4f2aca5
--- /dev/null
+++ b/packages/data-schemas/src/models/triggerLaneSequence.ts
@@ -0,0 +1,17 @@
+import { Model } from 'mongoose';
+import type { IAgentTriggerLaneSequenceDocument } from '~/types/triggerDelivery';
+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';
+import triggerLaneSequenceSchema from '~/schema/triggerLaneSequence';
+
+export function createAgentTriggerLaneSequenceModel(
+ mongoose: typeof import('mongoose'),
+): Model<IAgentTriggerLaneSequenceDocument> {
+ applyTenantIsolation(triggerLaneSequenceSchema);
+ return (
+ mongoose.models.AgentTriggerLaneSequence ||
+ mongoose.model<IAgentTriggerLaneSequenceDocument>(
+ 'AgentTriggerLaneSequence',
+ triggerLaneSequenceSchema,
+ )
+ );
+}
diff --git a/packages/data-schemas/src/models/triggerUserPurge.ts b/packages/data-schemas/src/models/triggerUserPurge.ts
new file mode 100644
index 0000000000..3e4837c7ad
--- /dev/null
+++ b/packages/data-schemas/src/models/triggerUserPurge.ts
@@ -0,0 +1,14 @@
+import { Model } from 'mongoose';
+import type { IAgentTriggerUserPurgeDocument } from '~/types/triggerDelivery';
+import { applyTenantIsolation } from '~/models/plugins/tenantIsolation';
+import triggerUserPurgeSchema from '~/schema/triggerUserPurge';
+
+export function createAgentTriggerUserPurgeModel(
+ mongoose: typeof import('mongoose'),
+): Model<IAgentTriggerUserPurgeDocument> {
+ applyTenantIsolation(triggerUserPurgeSchema);
+ return (
+ mongoose.models.AgentTriggerUserPurge ||
+ mongoose.model<IAgentTriggerUserPurgeDocument>('AgentTriggerUserPurge', triggerUserPurgeSchema)
+ );
+}
diff --git a/packages/data-schemas/src/schema/index.ts b/packages/data-schemas/src/schema/index.ts
index e6ac941d00..d9171d0099 100644
--- a/packages/data-schemas/src/schema/index.ts
+++ b/packages/data-schemas/src/schema/index.ts
@@ -32,3 +32,6 @@ export { default as groupSchema } from './group';
export { default as systemGrantSchema } from './systemGrant';
export { default as auditLogSchema } from './auditLog';
export { default as configSchema } from './config';
+export { default as triggerDeliverySchema } from './triggerDelivery';
+export { default as triggerLaneSequenceSchema } from './triggerLaneSequence';
+export { default as triggerUserPurgeSchema } from './triggerUserPurge';
diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts
new file mode 100644
index 0000000000..11f874c6bf
--- /dev/null
+++ b/packages/data-schemas/src/schema/triggerDelivery.ts
@@ -0,0 +1,71 @@
+import { Schema } from 'mongoose';
+import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';
+
+const failureSchema = new Schema(
+ {
+ code: { type: String, required: true, maxlength: 128 },
+ message: { type: String, required: true, maxlength: 2048 },
+ certainty: { type: String, enum: ['definite', 'ambiguous'], required: true },
+ retryable: { type: Boolean, required: true },
+ attemptedAt: { type: Date, required: true },
+ status: { type: Number },
+ },
+ { _id: false },
+);
+
+const historySchema = new Schema(
+ {
+ attempt: { type: Number, required: true, min: 1 },
+ outcome: { type: String, enum: ['succeeded', 'retry', 'dead'], required: true },
+ at: { type: Date, required: true },
+ workerId: { type: String, required: true },
+ error: { type: failureSchema },
+ },
+ { _id: false },
+);
+
+const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
+ {
+ deliveryKey: { type: String, required: true, maxlength: 128 },
+ fingerprint: { type: String, required: true, maxlength: 128 },
+ orderingKey: { type: String, required: true, maxlength: 128 },
+ // Zero is reserved for a staging row that is visible before sequence allocation.
+ laneSequence: { type: Number, required: true, min: 0 },
+ envelope: { type: Schema.Types.Mixed, required: true },
+ user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true },
+ tenantId: { type: String, index: true },
+ status: {
+ type: String,
+ enum: ['staging', 'pending', 'leased', 'succeeded', 'dead'],
+ required: true,
+ default: 'pending',
+ },
+ attempts: { type: Number, required: true, default: 0, min: 0 },
+ availableAt: { type: Date, required: true },
+ leaseBy: { type: String },
+ leaseUntil: { type: Date },
+ claimToken: { type: String },
+ lastError: { type: failureSchema },
+ result: { type: Schema.Types.Mixed },
+ history: { type: [historySchema], default: undefined },
+ settledAt: { type: Date },
+ expiresAt: { type: Date },
+ requeueCount: { type: Number, default: 0, min: 0 },
+ stagingRecoveryAt: { type: Date },
+ laneCleanupPendingAt: { type: Date },
+ },
+ { timestamps: true },
+);
+
+triggerDeliverySchema.index({ deliveryKey: 1 }, { unique: true });
+triggerDeliverySchema.index({ status: 1, availableAt: 1, createdAt: 1 });
+triggerDeliverySchema.index({ status: 1, leaseUntil: 1, createdAt: 1 });
+triggerDeliverySchema.index({ orderingKey: 1, status: 1, laneSequence: 1 });
+triggerDeliverySchema.index({ status: 1, updatedAt: -1 });
+triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });
+triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });
+// Only successful rows receive expiresAt. Dead letters remain available until
+// an operator explicitly requeues or removes them.
+triggerDeliverySchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });
+
+export default triggerDeliverySchema;
diff --git a/packages/data-schemas/src/schema/triggerLaneSequence.ts b/packages/data-schemas/src/schema/triggerLaneSequence.ts
new file mode 100644
index 0000000000..5a6bbb1369
--- /dev/null
+++ b/packages/data-schemas/src/schema/triggerLaneSequence.ts
@@ -0,0 +1,23 @@
+import { Schema } from 'mongoose';
+import type { IAgentTriggerLaneSequenceDocument } from '~/types/triggerDelivery';
+
+const triggerLaneSequenceSchema: Schema<IAgentTriggerLaneSequenceDocument> = new Schema(
+ {
+ _id: { type: String, required: true },
+ value: { type: Number, required: true, min: 1 },
+ user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true },
+ tenantId: { type: String, index: true },
+ tailDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },
+ publisherDeliveryId: { type: Schema.Types.ObjectId, ref: 'AgentTriggerDelivery' },
+ publisherStartedAt: { type: Date },
+ cleanupRequestedAt: { type: Date, index: true },
+ },
+ { timestamps: true },
+);
+
+// Publisher fields exist only while a lane admission is in flight. The sparse
+// index keeps the periodic recovery scan proportional to abandoned publishers
+// instead of all retained lane history (and is supported by DocumentDB 3.6+).
+triggerLaneSequenceSchema.index({ publisherStartedAt: 1 }, { sparse: true });
+
+export default triggerLaneSequenceSchema;
diff --git a/packages/data-schemas/src/schema/triggerUserPurge.ts b/packages/data-schemas/src/schema/triggerUserPurge.ts
new file mode 100644
index 0000000000..d627c5c998
--- /dev/null
+++ b/packages/data-schemas/src/schema/triggerUserPurge.ts
@@ -0,0 +1,13 @@
+import { Schema } from 'mongoose';
+import type { IAgentTriggerUserPurgeDocument } from '~/types/triggerDelivery';
+
+const triggerUserPurgeSchema: Schema<IAgentTriggerUserPurgeDocument> = new Schema(
+ {
+ _id: { type: Schema.Types.ObjectId, ref: 'User', required: true },
+ fenceStartedAt: { type: Date, required: true },
+ tenantId: { type: String, index: true },
+ },
+ { timestamps: true },
+);
+
+export default triggerUserPurgeSchema;
diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.ts
index 309ad54aa9..79a9a64195 100644
--- a/packages/data-schemas/src/schema/user.ts
+++ b/packages/data-schemas/src/schema/user.ts
@@ -131,6 +131,10 @@ const userSchema: Schema<IUser> = new Schema<IUser>(
type: Date,
default: null,
},
+ agentTriggerDeletionStartedAt: {
+ type: Date,
+ select: false,
+ },
personalization: {
type: {
memories: {
diff --git a/packages/data-schemas/src/types/index.ts b/packages/data-schemas/src/types/index.ts
index 7a96064f87..cecdab0a50 100644
--- a/packages/data-schemas/src/types/index.ts
+++ b/packages/data-schemas/src/types/index.ts
@@ -29,6 +29,7 @@ export * from './prompts';
/* Skills */
export * from './skill';
export * from './skillSync';
+export * from './triggerDelivery';
/* Access Control */
export * from './accessRole';
export * from './aclEntry';
diff --git a/packages/data-schemas/src/types/triggerDelivery.ts b/packages/data-schemas/src/types/triggerDelivery.ts
new file mode 100644
index 0000000000..e29fd2b7e6
--- /dev/null
+++ b/packages/data-schemas/src/types/triggerDelivery.ts
@@ -0,0 +1,105 @@
+import type { Document, Types } from 'mongoose';
+
+export type AgentTriggerDeliveryStatus = 'staging' | 'pending' | 'leased' | 'succeeded' | 'dead';
+export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';
+
+export interface AgentTriggerDeliveryFailure {
+ code: string;
+ message: string;
+ certainty: 'definite' | 'ambiguous';
+ retryable: boolean;
+ attemptedAt: Date;
+ status?: number;
+}
+
+export interface AgentTriggerDeliveryHistoryEntry {
+ attempt: number;
+ outcome: AgentTriggerDeliveryOutcome;
+ at: Date;
+ workerId: string;
+ error?: AgentTriggerDeliveryFailure;
+}
+
+export interface IAgentTriggerDelivery {
+ _id?: Types.ObjectId;
+ deliveryKey: string;
+ fingerprint: string;
+ orderingKey: string;
+ /** Monotonic sequence allocated while holding the lane publication fence. */
+ laneSequence: number;
+ envelope: unknown;
+ user: Types.ObjectId;
+ tenantId?: string;
+ status: AgentTriggerDeliveryStatus;
+ attempts: number;
+ availableAt: Date;
+ leaseBy?: string;
+ leaseUntil?: Date;
+ claimToken?: string;
+ lastError?: AgentTriggerDeliveryFailure;
+ result?: unknown;
+ history?: AgentTriggerDeliveryHistoryEntry[];
+ settledAt?: Date;
+ expiresAt?: Date;
+ requeueCount?: number;
+ /** Fairness cursor for bounded recovery of rows stranded before lane publication. */
+ stagingRecoveryAt?: Date;
+ /** Durable proof that successful settlement still owes lane cleanup publication. */
+ laneCleanupPendingAt?: Date;
+ createdAt?: Date;
+ updatedAt?: Date;
+}
+
+export interface IAgentTriggerDeliveryDocument
+ extends Omit<IAgentTriggerDelivery, '_id'>,
+ Document {}
+
+export interface AgentTriggerDeliveryRecord
+ extends Omit<IAgentTriggerDelivery, '_id' | 'createdAt'> {
+ id: string;
+ createdAt: Date;
+}
+
+export interface IAgentTriggerLaneSequence {
+ _id: string;
+ value: number;
+ user: Types.ObjectId;
+ tenantId?: string;
+ /** Latest delivery admitted to this lane. Used to reclaim inactive lane counters safely. */
+ tailDeliveryId?: Types.ObjectId;
+ /** Delivery currently owning the serialized sequence/publication step. */
+ publisherDeliveryId?: Types.ObjectId;
+ publisherStartedAt?: Date;
+ cleanupRequestedAt?: Date;
+ createdAt?: Date;
+ updatedAt?: Date;
+}
+
+export interface IAgentTriggerLaneSequenceDocument
+ extends Omit<IAgentTriggerLaneSequence, '_id'>,
+ Document<string> {}
+
+/** Durable proof that trigger payload cleanup must survive the deleting process. */
+export interface IAgentTriggerUserPurge {
+ _id: Types.ObjectId;
+ fenceStartedAt: Date;
+ tenantId?: string;
+ createdAt?: Date;
+ updatedAt?: Date;
+}
+
+export interface IAgentTriggerUserPurgeDocument
+ extends Omit<IAgentTriggerUserPurge, '_id'>,
+ Document {}
+
+export interface AgentTriggerDeliveryClaim extends AgentTriggerDeliveryRecord {
+ claimToken: string;
+ leaseBy: string;
+ leaseUntil: Date;
+ status: 'leased';
+}
+
+export interface AgentTriggerOrderingBlock {
+ availableAt: Date;
+ leaseUntil?: Date;
+}
diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.ts
index 2b91252a7a..169209cf4a 100644
--- a/packages/data-schemas/src/types/user.ts
+++ b/packages/data-schemas/src/types/user.ts
@@ -53,6 +53,8 @@ export interface IUser extends Document {
expiresAt?: Date;
termsAccepted?: boolean;
termsAcceptedAt?: Date | null;
+ /** Internal fence that prevents agent-trigger admission during account deletion. */
+ agentTriggerDeletionStartedAt?: Date;
personalization?: {
memories?: boolean;
statefulCodeEnvironment?: StatefulCodeEnvironment;