Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔕 perf: Back Off Idle Durable Agent Recovery Polls (#16303)
lia-by-librechat[bot]ffd3d1e139ce
diff --git a/api/server/experimental.js b/api/server/experimental.jsindex d6b610a595..45332738d4 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -709,6 +709,7 @@ if (cluster.isMaster) {address: server.address(),completionResultBatchSize:baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,+ idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling,});} catch (initErr) {logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.jsindex abe310b1ec..b1f05d4663 100644--- a/api/server/experimental.spec.js+++ b/api/server/experimental.spec.js@@ -95,6 +95,19 @@ describe('Experimental server configuration', () => {expect(listenIndex).toBeGreaterThan(eventRuntimeIndex);});+ it('passes the same idle recovery policy to both server startup paths', () => {+ const standard = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8');+ for (const [entrypoint, config] of [+ [source, 'baseAppConfig'],+ [standard, 'appConfig'],+ ]) {+ const start = entrypoint.indexOf('await initializeAgentTriggerService({');+ expect(start).toBeGreaterThan(-1);+ const call = entrypoint.slice(start, entrypoint.indexOf('});', start));+ expect(call).toContain(`idlePolling: ${config}?.endpoints?.agents?.eventDriven?.idlePolling`);+ }+ });+it('matches the standard server pre-authentication tenant routes', () => {expect(source).toContain("app.use('/oauth', preAuthTenantMiddleware, routes.oauth);");expect(source).toContain("app.use('/api/auth', preAuthTenantMiddleware, routes.auth);");diff --git a/api/server/index.js b/api/server/index.jsindex b4555e7b26..1d31f63b26 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -502,6 +502,7 @@ const startServer = async () => {address: server.address(),completionResultBatchSize:appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,+ idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling,});const scheduleEngineArmed = (await initializeScheduleEngine()) != null;scheduleEngineState = scheduleEngineArmed ? 'armed' : 'unavailable';diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.jsindex 0f2d81218e..7b58b54a9e 100644--- a/api/server/services/Agents/triggers.js+++ b/api/server/services/Agents/triggers.js@@ -64,7 +64,9 @@ service = createAgentTriggerService({const initializeAgentTriggerService = async (options) => {await service.initialize(options);- await queuedTurnLifecycle.initialize();+ await queuedTurnLifecycle.initialize({+ maxIdleIntervalMs: options?.idlePolling?.queuedTurnMaxIntervalMs,+ });};const stopAgentTriggerService = async () => {diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.jsindex 4c636f4692..df3a09b2d6 100644--- a/api/server/services/Agents/triggers.spec.js+++ b/api/server/services/Agents/triggers.spec.js@@ -80,6 +80,23 @@ describe('agent trigger service composition', () => {expect(supportsDetachedActionCompletion()).toBe(false);});+ it('passes the recovery policy to both delivery and queued-turn workers', async () => {+ const { initializeAgentTriggerService } = require('./triggers');+ const options = {+ address: 'local',+ idlePolling: {+ queuedTurnMaxIntervalMs: 60_000,+ maintenanceMaxIntervalMs: 90_000,+ deliveryMaxIntervalMs: 5_000,+ },+ };+ await initializeAgentTriggerService(options);+ expect(mockCreateAgentTriggerService.mock.results[0].value.initialize).toHaveBeenCalledWith(+ options,+ );+ expect(mockQueuedTurnLifecycle.initialize).toHaveBeenCalledWith({ maxIdleIntervalMs: 60_000 });+ });+it('injects the configured background completion batch size', async () => {const { initializeAgentTriggerService } = require('./triggers');await initializeAgentTriggerService({ address: 'local', completionResultBatchSize: 12 });diff --git a/packages/api/src/agents/checkpoints/deletion.integration.spec.ts b/packages/api/src/agents/checkpoints/deletion.integration.spec.tsindex 84d8b59eb9..5da8e089e3 100644--- a/packages/api/src/agents/checkpoints/deletion.integration.spec.ts+++ b/packages/api/src/agents/checkpoints/deletion.integration.spec.ts@@ -242,7 +242,9 @@ test('retains writer obligations, renewed revisions, and lookup failures', asyncawait deletion.remember(['child']);const getJobs = jest.fn().mockResolvedValue(['pending-host']);const reclaim = createCheckpointDeletionReclaimer(getJobs);- expect(await reclaim(25)).toBe(0);+ const activity = { found: false };+ expect(await reclaim(25, activity)).toBe(0);+ expect(activity.found).toBe(true);getJobs.mockRejectedValueOnce(new Error('job store unavailable'));await expect(reclaim(25)).rejects.toThrow('reclamation failed');getJobs.mockImplementationOnce(async () => {diff --git a/packages/api/src/agents/checkpoints/deletion.ts b/packages/api/src/agents/checkpoints/deletion.tsindex 074341dc2f..36fcd1a0cc 100644--- a/packages/api/src/agents/checkpoints/deletion.ts+++ b/packages/api/src/agents/checkpoints/deletion.ts@@ -225,9 +225,9 @@ export async function openCheckpointDeletion(/** Replay captured identities after topology deletion; never sweep new generations. */export function createCheckpointDeletionReclaimer(getOwnerJobs: (userId: string, tenantId?: string) => Promise<string[]>,-): (limit: number) => Promise<number> {+): (limit: number, activity?: { found: boolean }) => Promise<number> {let after: string | undefined;- return async (limit) => {+ return async (limit, activity) => {if (!Number.isSafeInteger(limit) || limit <= 0) throw new Error('Invalid reclamation limit');const db = mongoose.connection.db;if (!db || mongoose.connection.readyState !== 1)@@ -238,6 +238,7 @@ export function createCheckpointDeletionReclaimer(.sort({ _id: 1 }).limit(limit).toArray();+ if (activity != null && targets.length > 0) activity.found = true;after = targets.length === limit ? targets[targets.length - 1]._id : undefined;const jobsByOwner = new Map<string, Promise<string[]>>();diff --git a/packages/api/src/agents/queuedTurns.spec.ts b/packages/api/src/agents/queuedTurns.spec.tsindex adb41ec79d..b4157fd837 100644--- a/packages/api/src/agents/queuedTurns.spec.ts+++ b/packages/api/src/agents/queuedTurns.spec.ts@@ -843,6 +843,199 @@ describe('Agent queued-turn continuation', () => {});describe('Agent queued-turn delivery scheduling', () => {+ it('can retry initialization after a transient index setup failure', async () => {+ const ensureAgentQueuedTurnIndexes = jest+ .fn()+ .mockRejectedValueOnce(new Error('mongo unavailable'))+ .mockResolvedValue(undefined);+ const scheduler = createAgentQueuedTurnScheduler({+ methods: {+ ensureAgentQueuedTurnIndexes,+ findQueuedTurnsNeedingDelivery: jest.fn(async () => []),+ claimQueuedTurnsForAdmissionReconciliation: jest.fn(async () => []),+ } as unknown as AgentQueuedTurnMethods,+ enqueue: jest.fn(),+ getGenerationAdmissionEvidence: async () => null,+ });+ await expect(scheduler.initialize()).rejects.toThrow('mongo unavailable');+ await expect(scheduler.initialize()).resolves.toBeUndefined();+ expect(ensureAgentQueuedTurnIndexes).toHaveBeenCalledTimes(2);+ await scheduler.stop();+ });++ it('does not back off when discovery repairs reservations but returns no deliveries', async () => {+ jest.useFakeTimers();+ const findQueuedTurnsNeedingDelivery = jest.fn(+ async (_limit: number, activity: { found: boolean }) => {+ activity.found = true;+ return [];+ },+ );+ const scheduler = createAgentQueuedTurnScheduler({+ methods: {+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),+ findQueuedTurnsNeedingDelivery,+ claimQueuedTurnsForAdmissionReconciliation: jest.fn(async () => []),+ } as unknown as AgentQueuedTurnMethods,+ enqueue: jest.fn(),+ getGenerationAdmissionEvidence: async () => null,+ });+ try {+ await scheduler.initialize();+ await jest.advanceTimersByTimeAsync(30_000);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(2);+ } finally {+ await scheduler.stop();+ jest.useRealTimers();+ }+ });++ it('waits for sibling lease discovery after an error before allowing shutdown or another scan', async () => {+ jest.useFakeTimers();+ let release!: (turns: AgentQueuedTurnRecord[]) => void;+ const findQueuedTurnsNeedingDelivery = jest+ .fn()+ .mockRejectedValue(new Error('mongo unavailable'));+ const claimQueuedTurnsForAdmissionReconciliation = jest.fn(+ () =>+ new Promise<AgentQueuedTurnRecord[]>((resolve) => {+ release = resolve;+ }),+ );+ const scheduler = createAgentQueuedTurnScheduler({+ methods: {+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),+ findQueuedTurnsNeedingDelivery,+ claimQueuedTurnsForAdmissionReconciliation,+ } as unknown as AgentQueuedTurnMethods,+ enqueue: jest.fn(),+ getGenerationAdmissionEvidence: async () => null,+ });+ try {+ const starting = scheduler.initialize();+ await jest.advanceTimersByTimeAsync(120_000);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(1);+ let stopped = false;+ const stopping = scheduler.stop().then(() => {+ stopped = true;+ });+ await jest.advanceTimersByTimeAsync(0);+ expect(stopped).toBe(false);+ release([]);+ await Promise.all([starting, stopping]);+ expect(stopped).toBe(true);+ await jest.advanceTimersByTimeAsync(120_000);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(1);+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(1);+ } finally {+ await scheduler.stop();+ jest.useRealTimers();+ }+ });++ it('reduces empty Mongo discovery reads without losing bounded fallback polling', async () => {+ jest.useFakeTimers();+ try {+ const findQueuedTurnsNeedingDelivery = jest.fn(async () => []);+ const claimQueuedTurnsForAdmissionReconciliation = jest.fn(async () => []);+ const scheduler = createAgentQueuedTurnScheduler({+ methods: {+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),+ findQueuedTurnsNeedingDelivery,+ claimQueuedTurnsForAdmissionReconciliation,+ } as unknown as AgentQueuedTurnMethods,+ enqueue: jest.fn(),+ getGenerationAdmissionEvidence: async () => null,+ });+ await scheduler.initialize();+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(60_000);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(120_000);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(3);+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(3);+ await scheduler.stop();+ await jest.advanceTimersByTimeAsync(120_000);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(3);+ } finally {+ jest.useRealTimers();+ }+ });++ it('does not treat a deferred indeterminate admission as an empty queue', async () => {+ jest.useFakeTimers();+ jest.setSystemTime(new Date(NOW));+ try {+ const turn = {+ ...queuedTurn('queued-turn-deferred', 1),+ status: 'dead' as const,+ deliveryKey: 'delivery-deferred',+ admissionId: 'delivery-deferred',+ admissionProtocolVersion: 2 as const,+ terminalReceipt: {+ outcome: 'dead' as const,+ settledAt: new Date(NOW),+ failure: { code: 'ADMISSION_INDETERMINATE', message: 'pending evidence' },+ },+ };+ const claimQueuedTurnsForAdmissionReconciliation = jest.fn(async (input) => [+ {+ ...turn,+ reconciliationClaimId: input.claimId,+ reconciliationClaimBy: input.claimBy,+ },+ ]);+ const deferAgentQueuedTurnAdmissionReconciliation = jest.fn(async () => true);+ const scheduler = createAgentQueuedTurnScheduler({+ methods: {+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),+ findQueuedTurnsNeedingDelivery: jest.fn(async () => []),+ claimQueuedTurnsForAdmissionReconciliation,+ deferAgentQueuedTurnAdmissionReconciliation,+ } as unknown as AgentQueuedTurnMethods,+ enqueue: jest.fn(),+ getGenerationAdmissionEvidence: async () => null,+ });+ await scheduler.initialize();+ expect(deferAgentQueuedTurnAdmissionReconciliation).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(4_999);+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(1);+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(2);+ await scheduler.stop();+ } finally {+ jest.useRealTimers();+ }+ });++ it('wakes recovery after direct publication fails outside its own recovery pass', async () => {+ jest.useFakeTimers();+ try {+ const findQueuedTurnsNeedingDelivery = jest.fn(async () => []);+ const scheduler = createAgentQueuedTurnScheduler({+ methods: {+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),+ findQueuedTurnsNeedingDelivery,+ claimQueuedTurnsForAdmissionReconciliation: jest.fn(async () => []),+ reserveAgentQueuedTurnDelivery: jest.fn(async () => {+ throw new Error('mongo unavailable');+ }),+ } as unknown as AgentQueuedTurnMethods,+ enqueue: jest.fn(),+ getGenerationAdmissionEvidence: async () => null,+ });+ await scheduler.initialize();+ await expect(scheduler.schedule(queuedTurn('queued-turn-1', 1))).rejects.toThrow(+ 'mongo unavailable',+ );+ await jest.advanceTimersByTimeAsync(0);+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(2);+ await scheduler.stop();+ } finally {+ jest.useRealTimers();+ }+ });+function queuedTurn(id: string, sequence: number): AgentQueuedTurnRecord {const { claimId: _claimId, claimBy: _claimBy, claimUntil: _claimUntil, ...record } = claim();return {diff --git a/packages/api/src/agents/queuedTurns.ts b/packages/api/src/agents/queuedTurns.tsindex 6eeb6864f9..540e981b2d 100644--- a/packages/api/src/agents/queuedTurns.ts+++ b/packages/api/src/agents/queuedTurns.ts@@ -23,6 +23,7 @@ import type { AgentTriggerDeliveryFailure } from './triggers/engine';import { getAgentTriggerIdempotencyKey, parseAgentTriggerEnvelope } from './triggers/envelope';import { createAgentTriggerEnvelope } from './triggers/envelope';import { AgentTriggerExecutionError } from './triggers/host';+import { createIdleRecoveryLoop } from './recovery';export const AGENT_QUEUED_TURN_SOURCE = 'agent-queued-turn';const AGENT_QUEUED_TURN_EVENT = 'agent.queued-turn';@@ -32,6 +33,7 @@ const RECONCILIATION_LEASE_MS = 2 * 60 * 1000;const RECONCILIATION_BACKOFF_BASE_MS = 5_000;const RECONCILIATION_BACKOFF_MAX_MS = 5 * 60 * 1000;const DEFAULT_RECOVERY_INTERVAL_MS = 30_000;+const DEFAULT_RECOVERY_MAX_IDLE_INTERVAL_MS = 2 * 60_000;const DEFAULT_RECOVERY_LIMIT = 100;const MAX_FAILURE_CODE_LENGTH = 128;const MAX_FAILURE_MESSAGE_LENGTH = 2048;@@ -87,7 +89,7 @@ export interface AgentQueuedTurnSchedulerDeps {}export interface AgentQueuedTurnScheduler {- initialize: () => Promise<void>;+ initialize: (options?: { maxIdleIntervalMs?: number }) => Promise<void>;stop: () => Promise<void>;schedule: (turn: AgentQueuedTurnRecord) => Promise<string>;recover: () => Promise<number>;@@ -107,7 +109,7 @@ export interface AgentQueuedTurnLifecycle {rawSource: unknown,input: AgentQueuedTurnExecutionAdmission,) => Promise<boolean>;- initialize: () => Promise<void>;+ initialize: (options?: { maxIdleIntervalMs?: number }) => Promise<void>;stop: () => Promise<void>;schedule: (turn: AgentQueuedTurnRecord) => Promise<string>;cancel: (@@ -732,10 +734,12 @@ function createAgentQueuedTurnScheduler({recoveryIntervalMs = DEFAULT_RECOVERY_INTERVAL_MS,recoveryLimit = DEFAULT_RECOVERY_LIMIT,}: AgentQueuedTurnSchedulerDeps): AgentQueuedTurnScheduler {- let timer: NodeJS.Timeout | undefined;- let recovery: Promise<number> | undefined;+ let loop: ReturnType<typeof createIdleRecoveryLoop> | undefined;+ let initialization: Promise<void> | undefined;+ let stopped = false;+ let recovery: Promise<{ repaired: number; idle: boolean }> | undefined;- const schedule = async (turn: AgentQueuedTurnRecord): Promise<string> => {+ const publish = async (turn: AgentQueuedTurnRecord): Promise<string> => {const envelope = deliveryEnvelope(turn);const deliveryKey = getAgentTriggerIdempotencyKey(envelope);const reserved = await runAsSystem(() =>@@ -776,25 +780,43 @@ function createAgentQueuedTurnScheduler({return receipt.deliveryKey;};- const recover = (): Promise<number> => {+ const schedule = async (turn: AgentQueuedTurnRecord): Promise<string> => {+ try {+ return await publish(turn);+ } catch (error) {+ loop?.wake();+ throw error;+ }+ };++ const recoverPass = (): Promise<{ repaired: number; idle: boolean }> => {if (recovery != null) {return recovery;}const task = (async () => {const reconciliationNow = new Date();const reconciliationClaimId = randomUUID();- const [turns, quarantined] = await Promise.all([- runAsSystem(() => methods.findQueuedTurnsNeedingDelivery(recoveryLimit)),+ const activity = { found: false };+ const discoveries = await Promise.allSettled([+ runAsSystem(() => methods.findQueuedTurnsNeedingDelivery(recoveryLimit, activity)),runAsSystem(() =>- methods.claimQueuedTurnsForAdmissionReconciliation({- claimId: reconciliationClaimId,- claimBy: PROCESS_CLAIM_OWNER,- now: reconciliationNow,- leaseUntil: new Date(reconciliationNow.getTime() + RECONCILIATION_LEASE_MS),- limit: recoveryLimit,- }),+ methods.claimQueuedTurnsForAdmissionReconciliation(+ {+ claimId: reconciliationClaimId,+ claimBy: PROCESS_CLAIM_OWNER,+ now: reconciliationNow,+ leaseUntil: new Date(reconciliationNow.getTime() + RECONCILIATION_LEASE_MS),+ limit: recoveryLimit,+ },+ activity,+ ),),]);+ // A rejected discovery must not release the single-flight guard while+ // its sibling is still writing reconciliation leases in Mongo.+ const [deliveries, reconciliations] = discoveries;+ const turns = deliveries.status === 'fulfilled' ? deliveries.value : [];+ const quarantined = reconciliations.status === 'fulfilled' ? reconciliations.value : [];let repaired = 0;for (const turn of quarantined) {const deliveryKey = turn.deliveryKey;@@ -805,8 +827,11 @@ function createAgentQueuedTurnScheduler({) {continue;}- const defer = () =>- runAsSystem(() =>+ const defer = async () => {+ const availableAt = new Date(+ Date.now() + reconciliationBackoff(turn.reconciliationAttempts),+ );+ const deferred = await runAsSystem(() =>methods.deferAgentQueuedTurnAdmissionReconciliation({user: turn.user,...(turn.tenantId != null && { tenantId: turn.tenantId }),@@ -815,11 +840,13 @@ function createAgentQueuedTurnScheduler({deliveryKey,claimId: reconciliationClaimId,claimBy: PROCESS_CLAIM_OWNER,- availableAt: new Date(- Date.now() + reconciliationBackoff(turn.reconciliationAttempts),- ),+ availableAt,}),);+ if (deferred) {+ loop?.noteEligibleAt(availableAt);+ }+ };try {const isIndeterminate =turn.terminalReceipt?.outcome === 'dead' &&@@ -894,7 +921,7 @@ function createAgentQueuedTurnScheduler({}for (const turn of turns) {try {- await schedule(turn);+ await publish(turn);repaired += 1;} catch (error) {logger.warn(@@ -904,7 +931,11 @@ function createAgentQueuedTurnScheduler({);}}- return repaired;+ // Do not strand leases already acquired by the successful discovery just+ // because its independent sibling failed. Process them, then report failure.+ if (deliveries.status === 'rejected') throw deliveries.reason;+ if (reconciliations.status === 'rejected') throw reconciliations.reason;+ return { repaired, idle: !activity.found && turns.length === 0 && quarantined.length === 0 };})();recovery = task;void task.then(@@ -921,28 +952,48 @@ function createAgentQueuedTurnScheduler({);return task;};+ const recover = async (): Promise<number> => (await recoverPass()).repaired;return {schedule,recover,- initialize: async () => {- await runAsSystem(() => methods.ensureAgentQueuedTurnIndexes());- timer = setInterval(() => {- void recover().catch((error: unknown) => {- logger.warn('[agentQueuedTurns] Delivery recovery pass failed', error);+ initialize: (options = {}) => {+ if (initialization != null) {+ return initialization;+ }+ if (stopped) {+ return Promise.resolve();+ }+ const maxIdleIntervalMs =+ options.maxIdleIntervalMs ??+ Math.max(recoveryIntervalMs, DEFAULT_RECOVERY_MAX_IDLE_INTERVAL_MS);+ const next = (async () => {+ loop = createIdleRecoveryLoop({+ intervalMs: recoveryIntervalMs,+ maxIdleIntervalMs,+ scan: async () => (await recoverPass()).idle,+ onError: (error) =>+ logger.warn('[agentQueuedTurns] Delivery recovery pass failed', error),});- }, recoveryIntervalMs);- timer.unref?.();- await recover().catch((error: unknown) => {- logger.warn('[agentQueuedTurns] Initial delivery recovery pass failed', error);+ await runAsSystem(() => methods.ensureAgentQueuedTurnIndexes());+ if (!stopped) {+ await loop.start();+ }+ })();+ initialization = next.catch((error: unknown) => {+ if (!stopped) {+ initialization = undefined;+ loop = undefined;+ }+ throw error;});+ return initialization;},stop: async () => {- if (timer != null) {- clearInterval(timer);- timer = undefined;- }- await recovery;+ stopped = true;+ await loop?.stop();+ await initialization?.catch(() => undefined);+ await recovery?.catch(() => undefined);},};}diff --git a/packages/api/src/agents/recovery.spec.ts b/packages/api/src/agents/recovery.spec.tsnew file mode 100644index 0000000000..506d8c0625--- /dev/null+++ b/packages/api/src/agents/recovery.spec.ts@@ -0,0 +1,233 @@+import { createIdleRecoveryLoop } from './recovery';++describe('durable agent idle recovery', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ jest.setSystemTime(new Date('2026-09-01T00:00:00Z'));+ });++ afterEach(() => jest.useRealTimers());++ it('backs off only after empty passes and wakes without waiting for the idle ceiling', async () => {+ const scan = jest.fn(async () => true);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ await loop.start();+ expect(scan).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(59_999);+ expect(scan).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(1);+ expect(scan).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(120_000);+ expect(scan).toHaveBeenCalledTimes(3);+ loop.wake();+ await jest.advanceTimersByTimeAsync(0);+ expect(scan).toHaveBeenCalledTimes(4);+ await jest.advanceTimersByTimeAsync(59_999);+ expect(scan).toHaveBeenCalledTimes(4);+ await loop.stop();+ });++ it('never lets an in-flight empty result discard a wake or start overlapping scans', async () => {+ let release!: (idle: boolean) => void;+ const scan = jest.fn(+ () =>+ new Promise<boolean>((resolve) => {+ release = resolve;+ }),+ );+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ const first = loop.start();+ await jest.advanceTimersByTimeAsync(0);+ loop.wake();+ loop.wake();+ expect(scan).toHaveBeenCalledTimes(1);+ release(true);+ await first;+ await jest.advanceTimersByTimeAsync(0);+ expect(scan).toHaveBeenCalledTimes(2);+ release(true);+ await jest.advanceTimersByTimeAsync(0);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(scan).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(scan).toHaveBeenCalledTimes(3);+ release(false);+ await loop.stop();+ });++ it('scans at the base cadence after a failed or nonempty pass', async () => {+ const onError = jest.fn();+ const scan = jest+ .fn()+ .mockResolvedValueOnce(true)+ .mockRejectedValueOnce(new Error('mongo unavailable'))+ .mockResolvedValueOnce(false)+ .mockResolvedValue(true);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError,+ });+ await loop.start();+ await jest.advanceTimersByTimeAsync(60_000);+ expect(onError).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(scan).toHaveBeenCalledTimes(3);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(scan).toHaveBeenCalledTimes(4);+ await loop.stop();+ });++ it('scans at known eligibility times rather than sleeping past deferred work', async () => {+ const scan = jest.fn(async () => true);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ await loop.start();+ const now = Date.now();+ loop.noteEligibleAt(new Date(now + 14_000));+ loop.noteEligibleAt(new Date(now + 5_000));+ await jest.advanceTimersByTimeAsync(5_000);+ expect(scan).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(9_000);+ expect(scan).toHaveBeenCalledTimes(3);+ await loop.stop();+ });++ it('discovers work written by another replica without a process-local wake', async () => {+ let hasWork = false;+ const scan = jest.fn(async () => !hasWork);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ await loop.start();+ await jest.advanceTimersByTimeAsync(60_000);+ expect(scan).toHaveBeenCalledTimes(2);+ hasWork = true;+ await jest.advanceTimersByTimeAsync(119_999);+ expect(scan).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(1);+ expect(scan).toHaveBeenCalledTimes(3);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(scan).toHaveBeenCalledTimes(4);+ await loop.stop();+ });++ it('never postpones the bounded fallback scan when a later deadline is learned', async () => {+ const scan = jest.fn(async () => true);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ await loop.start();+ await jest.advanceTimersByTimeAsync(59_000);+ loop.noteEligibleAt(new Date(Date.now() + 120_000));+ await jest.advanceTimersByTimeAsync(1_000);+ expect(scan).toHaveBeenCalledTimes(2);+ await loop.stop();+ });++ it.each([false, true])(+ 'rescans for a deadline that expires in-flight (stop: %s)',+ async (stop) => {+ let release!: (idle: boolean) => void;+ const scan = jest+ .fn<Promise<boolean>, []>()+ .mockImplementationOnce(+ () =>+ new Promise((resolve) => {+ release = resolve;+ }),+ )+ .mockResolvedValue(true);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ const starting = loop.start();+ await jest.advanceTimersByTimeAsync(0);+ loop.noteEligibleAt(new Date(Date.now() + 5_000));+ loop.noteEligibleAt(new Date(Date.now() + 20_000));+ await jest.advanceTimersByTimeAsync(6_000);+ expect(scan).toHaveBeenCalledTimes(1);+ const stopping = stop ? loop.stop() : undefined;+ release(false);+ await starting;+ await stopping;+ await jest.advanceTimersByTimeAsync(0);+ expect(scan).toHaveBeenCalledTimes(stop ? 1 : 2);+ await jest.advanceTimersByTimeAsync(14_000);+ expect(scan).toHaveBeenCalledTimes(stop ? 1 : 3);+ await loop.stop();+ expect(jest.getTimerCount()).toBe(0);+ },+ );++ it('does not rescan or reset its idle timer when start is called twice', async () => {+ const scan = jest.fn(async () => true);+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ await loop.start();+ await jest.advanceTimersByTimeAsync(59_000);+ await loop.start();+ expect(scan).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(1_000);+ expect(scan).toHaveBeenCalledTimes(2);+ await loop.stop();+ });++ it('waits for its active scan on shutdown and never rearms a timer', async () => {+ let release!: (value: boolean) => void;+ const scan = jest.fn(+ () =>+ new Promise<boolean>((resolve) => {+ release = resolve;+ }),+ );+ const loop = createIdleRecoveryLoop({+ intervalMs: 30_000,+ maxIdleIntervalMs: 120_000,+ scan,+ onError: jest.fn(),+ });+ const started = loop.start();+ await jest.advanceTimersByTimeAsync(0);+ let stopped = false;+ const stopping = loop.stop().then(() => {+ stopped = true;+ });+ loop.wake();+ expect(stopped).toBe(false);+ release(true);+ await Promise.all([started, stopping]);+ await jest.advanceTimersByTimeAsync(300_000);+ expect(stopped).toBe(true);+ expect(scan).toHaveBeenCalledTimes(1);+ expect(jest.getTimerCount()).toBe(0);+ });+});diff --git a/packages/api/src/agents/recovery.ts b/packages/api/src/agents/recovery.tsnew file mode 100644index 0000000000..93d3ca1b5d--- /dev/null+++ b/packages/api/src/agents/recovery.ts@@ -0,0 +1,164 @@+export interface IdleRecoveryLoopOptions {+ intervalMs: number;+ maxIdleIntervalMs: number;+ /** True only after a completed scan confirmed no eligible work and no failures. */+ scan: () => Promise<boolean>;+ onError: (error: unknown) => void;+}++/** Mongo remains the authority: wakes are only hints, and every replica still scans+ * within the idle ceiling so a crashed producer or a missed local wake is recovered. */+export function createIdleRecoveryLoop({+ intervalMs,+ maxIdleIntervalMs,+ scan,+ onError,+}: IdleRecoveryLoopOptions) {+ if (!Number.isSafeInteger(intervalMs) || intervalMs <= 0) {+ throw new TypeError('Recovery interval must be a positive integer');+ }+ if (!Number.isSafeInteger(maxIdleIntervalMs) || maxIdleIntervalMs < intervalMs) {+ throw new TypeError('Maximum idle recovery interval must be at least the recovery interval');+ }++ let started = false;+ let stopped = false;+ let idleStreak = 0;+ let nextPollAt = 0;+ let wakePending = false;+ let generation = 0;+ let timer: NodeJS.Timeout | undefined;+ let active: Promise<void> | undefined;+ const deadlines: number[] = [];+ const MAX_DEADLINES = 64;++ const schedule = (): void => {+ if (!started || stopped || active != null) {+ return;+ }+ if (timer != null) {+ clearTimeout(timer);+ }+ const now = Date.now();+ let delay = Math.max(0, nextPollAt - now);+ if (deadlines.length > 0) {+ delay = Math.min(delay, Math.max(0, deadlines[0] - now));+ }+ timer = setTimeout(() => {+ timer = undefined;+ void runNow();+ }, delay);+ timer.unref?.();+ };++ const runNow = (): Promise<void> => {+ if (stopped || !started) {+ return Promise.resolve();+ }+ if (active != null) {+ return active;+ }+ if (timer != null) {+ clearTimeout(timer);+ timer = undefined;+ }+ // This scan can cover deadlines already due when it starts. Deadlines+ // learned or becoming due while it runs must survive for a follow-up scan.+ const now = Date.now();+ while (deadlines.length > 0 && deadlines[0] <= now) {+ deadlines.shift();+ }+ const before = generation;+ const current = Promise.resolve()+ .then(scan)+ .then(+ (idle) => {+ idleStreak = idle && before === generation ? idleStreak + 1 : 0;+ },+ (error: unknown) => {+ idleStreak = 0;+ onError(error);+ },+ )+ .finally(() => {+ if (active === current) {+ active = undefined;+ }+ if (stopped) {+ return;+ }+ nextPollAt =+ Date.now() + Math.min(intervalMs * 2 ** Math.min(idleStreak, 30), maxIdleIntervalMs);+ if (wakePending) {+ wakePending = false;+ queueMicrotask(() => void runNow());+ } else {+ schedule();+ }+ });+ active = current;+ return current;+ };++ return {+ start: (): Promise<void> => {+ if (stopped) {+ return Promise.resolve();+ }+ if (started) {+ return active ?? Promise.resolve();+ }+ started = true;+ return runNow();+ },+ /** Coalesce notifications while a scan is in progress; none can be erased+ * by that scan's earlier empty result. */+ wake: (): void => {+ if (stopped) {+ return;+ }+ generation += 1;+ idleStreak = 0;+ if (active != null) {+ wakePending = true;+ } else if (started) {+ void runNow();+ }+ },+ noteEligibleAt: (at: Date): void => {+ const time = at.getTime();+ if (!Number.isFinite(time) || stopped) {+ return;+ }+ if (time <= Date.now()) {+ if (started) {+ generation += 1;+ idleStreak = 0;+ if (active != null) {+ wakePending = true;+ } else {+ void runNow();+ }+ }+ return;+ }+ const index = deadlines.findIndex((value) => value >= time);+ if (index !== -1 && deadlines[index] === time) {+ return;+ }+ deadlines.splice(index < 0 ? deadlines.length : index, 0, time);+ if (deadlines.length > MAX_DEADLINES) {+ deadlines.pop();+ }+ schedule();+ },+ stop: async (): Promise<void> => {+ stopped = true;+ if (timer != null) {+ clearTimeout(timer);+ timer = undefined;+ }+ await active;+ },+ };+}diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.mdindex 93abb3f4bb..faef699ef5 100644--- a/packages/api/src/agents/triggers/README.md+++ b/packages/api/src/agents/triggers/README.md@@ -47,6 +47,7 @@ await enqueueAgentTrigger(## Guarantees- Mongo owns queue state, leases, retry history, and dead letters across restarts and replicas.+- Each replica retains bounded Mongo polling even when idle. See **Idle recovery** below for wake-ups, deadlines, and configuration.- A fresh token fences every claim, including reclaims by the same process.- A delivery is at-least-once. Fire, continue, and steer admission reuse the envelope's stable idempotencyidentity, so ambiguous retries do not duplicate accepted work.@@ -71,6 +72,37 @@ await enqueueAgentTrigger(`getAgentTriggerDeadLetters` and `requeueAgentTrigger` are intentionally trusted in-processoperations. Exposing them through an admin API requires a separate authorization and audit layer.+## Idle recovery++Queued-turn and maintenance scans start at a 30-second cadence and double the next idle wait+up to two minutes after confirmed-empty discovery. Inspected reservations, reconciliation+candidates, outstanding cleanup markers, full legacy-receipt pages, and failed scans do not+count as empty. A failed discovery waits for its sibling Mongo operation to settle before a+new pass or shutdown can complete; successfully acquired work still gets processed.++Local failed publication and requeue, unfinished terminal finalization, and purge markers wake+maintenance without waiting for an idle timer. A committed root success or dead letter remains+authoritative when its inline cleanup fails. Healthy terminal writes do not wake a full sweep.+Known reconciliation deadlines cap the next wait, including deadlines that expire while a scan+is in flight. Notifications coalesce behind one active scan; shutdown prevents a follow-up.++Every replica still scans without notifications, so a crashed producer or a missed cross-replica+wake cannot strand durable work. The cap limits **idle sleep**, not end-to-end recovery time:+scan duration, pagination, active leases, and persistence failures can add time. Delivery claims+retain their separate 15-second idle cap and immediate local enqueue/requeue wake-ups.++Configure `endpoints.agents.eventDriven.idlePolling` in `librechat.yaml`, for either standard or+experimental clustered startup:++| Setting | Default (ms) | Allowed range (ms) |+| --- | ---: | ---: |+| `queuedTurnMaxIntervalMs` | 120000 | 30000–300000 |+| `maintenanceMaxIntervalMs` | 120000 | 30000–300000 |+| `deliveryMaxIntervalMs` | 15000 | 1000–300000 |++Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data+migration is needed; optional activity reporting preserves the existing numeric/boolean results.+## Remote event ingressAuthenticated controllers and source adapters can enqueue the same durable envelope throughdiff --git a/packages/api/src/agents/triggers/delivery.integration.spec.ts b/packages/api/src/agents/triggers/delivery.integration.spec.tsindex bc78de4fa1..364edf0be9 100644--- a/packages/api/src/agents/triggers/delivery.integration.spec.ts+++ b/packages/api/src/agents/triggers/delivery.integration.spec.ts@@ -4,6 +4,7 @@ import { createMethods, createModels } from '@librechat/data-schemas';import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service';import type { AgentTriggerFetch } from './host';import { __resetShutdownStateForTests } from '../../app/shutdown';+import { prepareAgentTriggerDelivery } from './delivery';import { createAgentTriggerEnvelope } from './envelope';import { createAgentTriggerService } from './service';@@ -83,6 +84,79 @@ async function eventuallySucceeded(deliveryKey: string) {}describe('durable trigger delivery integration', () => {+ it('recovers an abandoned publication across two idle workers without a producer wake', async () => {+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(+ async () =>+ new Response(+ JSON.stringify({+ status: 'started',+ streamId: 'cross-replica',+ conversationId: 'cross-replica',+ generationCreatedAt: 25,+ }),+ { status: 200, headers: { 'content-type': 'application/json' } },+ ),+ );+ const bundles = [createMethods(mongoose), createMethods(mongoose)];+ const scans = bundles.map((methods) =>+ jest.spyOn(methods, 'recoverAgentTriggerLanePublications'),+ );+ const workers = bundles.map((methods) =>+ createAgentTriggerService({+ methods: methods as typeof methods & AgentTriggerDeliveryPersistence,+ fetch: fetcher,+ mintToken: () => 'trigger-token',+ purgeRecoveryIntervalMs: 20,+ deliveryOptions: {+ concurrency: 1,+ tickMs: 5,+ maxIdleTickMs: 40,+ },+ }),+ );+ const until = async (condition: () => boolean | Promise<boolean>) => {+ const deadline = Date.now() + 5_000;+ while (!(await condition())) {+ if (Date.now() >= deadline) throw new Error('Timed out waiting for cross-replica recovery');+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ };+ try {+ for (const worker of workers)+ await worker.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ idlePolling: { maintenanceMaxIntervalMs: 80 },+ });+ await until(() => scans.every((scan) => scan.mock.calls.length >= 3));+ const prepared = prepareAgentTriggerDelivery(envelope(), { orderingKey: 'crashed-producer' });+ // A producer died after its staging write. Neither worker receives enqueue()+ // or wake(); both must discover and fence publication via their Mongo fallback.+ await mongoose.models.AgentTriggerDelivery.create({+ ...prepared,+ laneSequence: 0,+ status: 'staging',+ attempts: 0,+ requeueCount: 0,+ claimAvailableAt: prepared.availableAt,+ stagingRecoveryAt: new Date(),+ });+ await until(+ async () =>+ (await bundles[0].getAgentTriggerDelivery(prepared.deliveryKey))?.status === 'succeeded',+ );+ expect(fetcher).toHaveBeenCalledTimes(1);+ const delivered = await bundles[1].getAgentTriggerDelivery(prepared.deliveryKey);+ expect(delivered).toMatchObject({+ status: 'succeeded',+ attempts: 1,+ history: [{ outcome: 'succeeded' }],+ });+ } finally {+ await Promise.all(workers.map((worker) => worker.stop()));+ scans.forEach((scan) => scan.mockRestore());+ }+ });+it('moves a trusted envelope through Mongo, the lease worker, and host admission', async () => {const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(async () =>diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsindex f3b6b4c103..8d571b0f92 100644--- a/packages/api/src/agents/triggers/service.delivery.spec.ts+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -7,9 +7,11 @@ import {SYSTEM_TENANT_ID,} from '@librechat/data-schemas';import type { AgentTriggerDeliveryPersistence, AgentTriggerStoredRecord } from './service';+import type { AgentTriggerDeliveryStore } from './engine';import { AgentTriggerServiceUnavailableError, createAgentTriggerService } from './service';import { __resetShutdownStateForTests } from '../../app/shutdown';import { createAgentTriggerEnvelope } from './envelope';+import * as deliveryEngineModule from './engine';jest.mock('@librechat/data-schemas', () => {const actual = jest.requireActual('@librechat/data-schemas');@@ -321,6 +323,183 @@ describe('durable agent trigger service', () => {expect(deleteAgentTriggerDeliveriesByUser).toHaveBeenCalledWith('507f1f77bcf86cd799439011');});+ it.each(['complete', 'dead', 'retire'] as const)(+ 'wakes for unfinished %s finalization, but not healthy terminal writes',+ async (operation) => {+ jest.useFakeTimers();+ let service: ReturnType<typeof createAgentTriggerService> | undefined;+ try {+ let store!: AgentTriggerDeliveryStore;+ const create = deliveryEngineModule.createAgentTriggerDeliveryEngine;+ jest+ .spyOn(deliveryEngineModule, 'createAgentTriggerDeliveryEngine')+ .mockImplementation((deps, options) => {+ store = deps.store;+ return create(deps, options);+ });+ let unfinished = false;+ const terminal = jest.fn(async (_input: unknown, recovery?: { required: boolean }) => {+ if (recovery != null) recovery.required = unfinished;+ return true;+ });+ const methods = deliveryMethods({+ completeAgentTriggerDelivery: terminal,+ deadLetterAgentTriggerDelivery: terminal,+ retireAgentTriggerDelivery: terminal,+ });+ service = createAgentTriggerService({ methods, deliveryOptions: { tickMs: 300_000 } });+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });+ await jest.advanceTimersByTimeAsync(180_000);+ const count = (methods.recoverAgentTriggerBatchReceipts as jest.Mock).mock.calls.length;+ const input = {+ id: 'row',+ workerId: 'worker',+ claimToken: 'token',+ attempt: 1,+ settledAt: new Date(),+ };+ const finish = () => {+ if (operation === 'complete') {+ return store.complete({+ ...input,+ result: { mode: 'fire', status: 'started', conversationId: 'conversation-1' },+ });+ }+ if (operation === 'dead') {+ return store.dead({+ ...input,+ error: {+ code: 'FAILED',+ message: 'failed',+ retryable: false,+ certainty: 'definite',+ attemptedAt: new Date(),+ },+ });+ }+ return service!.retire('delivery-key', 'source', 'cancelled');+ };+ await expect(finish()).resolves.toBe(true);+ await jest.advanceTimersByTimeAsync(0);+ expect(methods.recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(count);+ unfinished = true;+ await expect(finish()).resolves.toBe(true);+ await jest.advanceTimersByTimeAsync(0);+ expect(methods.recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(count + 1);+ expect(getTenantId()).not.toBe(SYSTEM_TENANT_ID);+ } finally {+ await service?.stop();+ jest.restoreAllMocks();+ jest.useRealTimers();+ }+ },+ );++ it.each(['enqueue', 'requeue'] as const)(+ 'wakes maintenance after an ambiguous %s write failure',+ async (operation) => {+ jest.useFakeTimers();+ let service: ReturnType<typeof createAgentTriggerService> | undefined;+ try {+ const methods = deliveryMethods();+ (methods.enqueueAgentTriggerDelivery as jest.Mock).mockRejectedValue(+ new Error('publication interrupted'),+ );+ (methods.requeueAgentTriggerDelivery as jest.Mock).mockRejectedValue(+ new Error('publication interrupted'),+ );+ service = createAgentTriggerService({ methods, deliveryOptions: { tickMs: 300_000 } });+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });+ await jest.advanceTimersByTimeAsync(180_000);+ const count = (methods.recoverAgentTriggerLanePublications as jest.Mock).mock.calls.length;+ await expect(+ operation === 'enqueue' ? service.enqueue(envelope()) : service.requeue('row'),+ ).rejects.toThrow('publication interrupted');+ await jest.advanceTimersByTimeAsync(0);+ expect(methods.recoverAgentTriggerLanePublications).toHaveBeenCalledTimes(count + 1);+ } finally {+ await service?.stop();+ jest.useRealTimers();+ }+ },+ );++ it('backs off empty maintenance scans and wakes for new cleanup work', async () => {+ jest.useFakeTimers();+ try {+ const recoverAgentTriggerUserPurges = jest.fn(async () => 0);+ const methods = deliveryMethods({ recoverAgentTriggerUserPurges });+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 300_000 },+ });+ await service.initialize({+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },+ });+ await jest.advanceTimersByTimeAsync(0);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(60_000);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(2);+ await jest.advanceTimersByTimeAsync(120_000);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(3);+ await service.prepareUserPurge('507f1f77bcf86cd799439011', new Date());+ await jest.advanceTimersByTimeAsync(0);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(4);+ await service.stop();+ await jest.advanceTimersByTimeAsync(120_000);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(4);+ } finally {+ jest.useRealTimers();+ }+ });++ it('keeps the base cadence when a recovery marker was inspected but no repair completed', async () => {+ jest.useFakeTimers();+ try {+ const recoverAgentTriggerBatchReceipts = jest.fn(+ async (_limit?: number, activity?: { found: boolean }) => {+ if (activity != null) activity.found = true;+ return 0;+ },+ );+ const service = createAgentTriggerService({+ methods: deliveryMethods({ recoverAgentTriggerBatchReceipts }),+ deliveryOptions: { concurrency: 1, tickMs: 300_000 },+ });+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });+ await jest.advanceTimersByTimeAsync(0);+ expect(recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(2);+ await service.stop();+ } finally {+ jest.useRealTimers();+ }+ });++ it('does not mistake an isolated maintenance failure for an empty scan', async () => {+ jest.useFakeTimers();+ try {+ const recoverAgentTriggerUserPurges = jest+ .fn()+ .mockRejectedValueOnce(new Error('mongo unavailable'))+ .mockResolvedValue(0);+ const methods = deliveryMethods({ recoverAgentTriggerUserPurges });+ const service = createAgentTriggerService({+ methods,+ deliveryOptions: { concurrency: 1, tickMs: 300_000 },+ });+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });+ await jest.advanceTimersByTimeAsync(0);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(30_000);+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(2);+ await service.stop();+ } finally {+ jest.useRealTimers();+ }+ });+it('arms and disarms post-commit purge recovery in system context', async () => {const prepareAgentTriggerUserPurge = jest.fn(async () => {expect(getTenantId()).toBe(SYSTEM_TENANT_ID);@@ -385,7 +564,11 @@ describe('durable agent trigger service', () => {address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },});- expect(expireLegacyAgentEventActorReceipts).toHaveBeenCalledWith(expect.any(Date), 17);+ expect(expireLegacyAgentEventActorReceipts).toHaveBeenCalledWith(+ expect.any(Date),+ 17,+ expect.objectContaining({ found: false }),+ );await service.stop();});@@ -405,7 +588,10 @@ describe('durable agent trigger service', () => {deliveryOptions: { concurrency: 1, tickMs: 60_000 },});await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });- expect(reclaimCheckpointDeletions).toHaveBeenCalledWith(17);+ expect(reclaimCheckpointDeletions).toHaveBeenCalledWith(+ 17,+ expect.objectContaining({ found: false }),+ );let stopped = false;const stop = service.stop().then(() => {stopped = true;@@ -545,6 +731,7 @@ describe('durable agent trigger service', () => {reason: 'result unavailable',onlyIfUnclaimed: true,}),+ { required: false },);await expect(service.retire('trigger_1', 'background-tool-completion', 'dead recovery', {@@ -553,6 +740,7 @@ describe('durable agent trigger service', () => {).resolves.toBe(true);expect(retireAgentTriggerDelivery).toHaveBeenLastCalledWith(expect.objectContaining({ onlyIfDead: true }),+ { required: false },);await service.stop();});diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsindex 18f7b32c69..a37e1821f3 100644--- a/packages/api/src/agents/triggers/service.ts+++ b/packages/api/src/agents/triggers/service.ts@@ -32,16 +32,23 @@ import { prepareAgentTriggerDelivery } from './delivery';import { selfOriginFromAddress } from '../../app/origin';import { createAgentTriggerExecutionHost } from './host';import { parseAgentTriggerEnvelope } from './envelope';+import { createIdleRecoveryLoop } from '../recovery';export const AGENT_TRIGGER_TOKEN_TTL = '60s';const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;const DEFAULT_USER_DRAIN_POLL_MS = 100;const DEFAULT_PURGE_RECOVERY_INTERVAL_MS = 30_000;+const DEFAULT_PURGE_RECOVERY_MAX_IDLE_INTERVAL_MS = 2 * 60_000;const DEFAULT_PURGE_RECOVERY_LIMIT = 25;export interface AgentTriggerServiceOptions {completionResultBatchSize?: number;address?: BoundAddress | string | null;+ idlePolling?: {+ queuedTurnMaxIntervalMs?: number;+ maintenanceMaxIntervalMs?: number;+ deliveryMaxIntervalMs?: number;+ };}export interface AgentTriggerServiceDeps {@@ -57,7 +64,7 @@ export interface AgentTriggerServiceDeps {userDrainPollMs?: number;purgeRecoveryIntervalMs?: number;purgeRecoveryLimit?: number;- reclaimCheckpointDeletions?: (limit: number) => Promise<number>;+ reclaimCheckpointDeletions?: (limit: number, activity?: { found: boolean }) => Promise<number>;supportsDetachedActionCompletion?: () => boolean;settleSourceBeforeDeadLetter?: AgentTriggerDeliveryEngineDeps['settleSourceBeforeDeadLetter'];}@@ -111,14 +118,14 @@ export interface AgentTriggerDeliveryPersistence {releaseAgentTriggerDelivery: AgentTriggerDeliveryStore['release'];beginAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['beginAttempt'];deferAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['defer'];- completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];+ completeAgentTriggerDelivery: AgentTriggerDeliveryMethods['completeAgentTriggerDelivery'];retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];- deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];+ deadLetterAgentTriggerDelivery: AgentTriggerDeliveryMethods['deadLetterAgentTriggerDelivery'];getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;getAgentTriggerDeliveryStatus: (deliveryKey: string,@@ -132,17 +139,30 @@ export interface AgentTriggerDeliveryPersistence {availableAt: Date,) => Promise<AgentTriggerStoredRecord | null>;countActiveAgentTriggerDeliveriesByUser: (userId: string, now: Date) => Promise<number>;- recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;- recoverAgentTriggerBatchReceipts: (limit?: number) => Promise<number>;- reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;+ recoverAgentTriggerLanePublications: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;+ recoverAgentTriggerBatchReceipts: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;+ reclaimInactiveAgentTriggerLanes: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;prepareAgentTriggerUserPurge: (userId: string,fenceStartedAt: Date,tenantId?: string,) => Promise<void>;cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;- recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;- expireLegacyAgentEventActorReceipts?: (now: Date, limit?: number) => Promise<number>;+ recoverAgentTriggerUserPurges: (limit?: number, activity?: { found: boolean }) => Promise<number>;+ expireLegacyAgentEventActorReceipts?: (+ now: Date,+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;deleteAgentTriggerDeliveriesByUser: (userId: string) => Promise<void>;}@@ -193,9 +213,29 @@ export interface AgentTriggerService {purgeUser: (userId: string) => Promise<void>;}+/** A producer can commit durable state before an inline finalizer fails. Keep+ * its authoritative result intact, but wake maintenance for the remaining marker.+ * Rejections are uncertain commits, so they also request recovery. Healthy writes+ * do not turn every delivered event into a full maintenance sweep. */+async function withMaintenanceRecovery<T>(+ operation: (recovery: { required: boolean }) => Promise<T>,+ wake: () => void,+): Promise<T> {+ const recovery = { required: false };+ try {+ return await operation(recovery);+ } catch (error) {+ recovery.required = true;+ throw error;+ } finally {+ if (recovery.required) wake();+ }+}+function createDeliveryStore(methods: AgentTriggerDeliveryPersistence,supportsDetachedActionCompletion: () => boolean,+ wakeMaintenance: () => void,): AgentTriggerDeliveryStore {return {claimNext: (input) =>@@ -215,9 +255,17 @@ function createDeliveryStore(release: methods.releaseAgentTriggerDelivery,beginAttempt: methods.beginAgentTriggerDeliveryAttempt,defer: methods.deferAgentTriggerDeliveryAttempt,- complete: methods.completeAgentTriggerDelivery,+ complete: (input) =>+ withMaintenanceRecovery(+ (recovery) => methods.completeAgentTriggerDelivery(input, recovery),+ wakeMaintenance,+ ),retry: methods.retryAgentTriggerDelivery,- dead: methods.deadLetterAgentTriggerDelivery,+ dead: (input) =>+ withMaintenanceRecovery(+ (recovery) => methods.deadLetterAgentTriggerDelivery(input, recovery),+ wakeMaintenance,+ ),};}@@ -277,8 +325,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Alet backgroundCompletionResultBatchSize = 8;let deliveryEngine: AgentTriggerDeliveryEngine | undefined;let initializePromise: Promise<void> | undefined;- let purgeRecoveryPromise: Promise<void> | undefined;- let purgeRecoveryTimer: NodeJS.Timeout | undefined;+ let purgeRecoveryPromise: Promise<boolean> | undefined;+ let purgeRecoveryLoop: ReturnType<typeof createIdleRecoveryLoop> | undefined;let deliveryReady = false;let stopping = false;const isPrincipalActive = deps.isPrincipalActive;@@ -362,9 +410,9 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}};- const recoverPurges = (): Promise<void> => {+ const recoverPurges = (): Promise<boolean> => {if (deps.methods == null || stopping) {- return Promise.resolve();+ return Promise.resolve(false);}if (purgeRecoveryPromise != null) {return purgeRecoveryPromise;@@ -378,14 +426,20 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A* half-recovered batch clears a request that a later successful recovery* can no longer re-arm, retaining the lane permanently — so reclamation* still waits for a batch-recovery pass that did not fail. */- const isolated = (label: string, run: () => Promise<number>): Promise<number> =>- run().catch((error) => {+ let failed = false;+ const activity = { found: false };+ const isolated = async (label: string, run: () => Promise<number>): Promise<number> => {+ try {+ return await run();+ } catch (error) {+ failed = true;logger.error(`[agent-triggers] durable delivery maintenance step failed (${label}):`,error,);return 0;- });+ }+ };const current = runAsSystem(async () => {const [purgedUsers,@@ -394,13 +448,16 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): AexpiredLegacyActorReceipts,retiredCheckpointDeletions,] = await Promise.all([- isolated('user purges', () => methods.recoverAgentTriggerUserPurges(purgeRecoveryLimit)),+ isolated('user purges', () =>+ methods.recoverAgentTriggerUserPurges(purgeRecoveryLimit, activity),+ ),isolated('lane publications', () =>- methods.recoverAgentTriggerLanePublications(purgeRecoveryLimit),+ methods.recoverAgentTriggerLanePublications(purgeRecoveryLimit, activity),),- methods.recoverAgentTriggerBatchReceipts(purgeRecoveryLimit).then(+ methods.recoverAgentTriggerBatchReceipts(purgeRecoveryLimit, activity).then((count) => ({ succeeded: true as const, count }),(error) => {+ failed = true;logger.error('[agent-triggers] durable delivery maintenance step failed (batch receipts):',error,@@ -411,18 +468,22 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Aisolated('legacy actor receipts',() =>- methods.expireLegacyAgentEventActorReceipts?.(new Date(), purgeRecoveryLimit) ??- Promise.resolve(0),+ methods.expireLegacyAgentEventActorReceipts?.(+ new Date(),+ purgeRecoveryLimit,+ activity,+ ) ?? Promise.resolve(0),),isolated('checkpoint deletion evidence',- () => deps.reclaimCheckpointDeletions?.(purgeRecoveryLimit) ?? Promise.resolve(0),+ () =>+ deps.reclaimCheckpointDeletions?.(purgeRecoveryLimit, activity) ?? Promise.resolve(0),),]);const recoveredBatches = batchRecovery.count;const reclaimedLanes = batchRecovery.succeeded? await isolated('lane reclamation', () =>- methods.reclaimInactiveAgentTriggerLanes(purgeRecoveryLimit),+ methods.reclaimInactiveAgentTriggerLanes(purgeRecoveryLimit, activity),): 0;if (publishedLanes > 0) {@@ -445,9 +506,20 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): AretiredCheckpointDeletions,});}+ return (+ !failed &&+ !activity.found &&+ purgedUsers === 0 &&+ publishedLanes === 0 &&+ recoveredBatches === 0 &&+ reclaimedLanes === 0 &&+ expiredLegacyActorReceipts === 0 &&+ retiredCheckpointDeletions === 0+ );}).catch((error) => {logger.error('[agent-triggers] durable delivery maintenance failed:', error);+ return false;}).finally(() => {if (purgeRecoveryPromise === current) {@@ -459,21 +531,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A};const startPurgeRecovery = (): void => {- if (purgeRecoveryTimer != null) {- return;- }- void recoverPurges();- purgeRecoveryTimer = setInterval(() => void recoverPurges(), purgeRecoveryIntervalMs);- purgeRecoveryTimer.unref();+ void purgeRecoveryLoop?.start();};const stop = async (): Promise<void> => {stopping = true;deliveryReady = false;- if (purgeRecoveryTimer != null) {- clearInterval(purgeRecoveryTimer);- purgeRecoveryTimer = undefined;- }+ await purgeRecoveryLoop?.stop();await initializePromise?.catch(() => undefined);await deliveryEngine?.stop();await purgeRecoveryPromise?.catch(() => undefined);@@ -506,6 +570,15 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): Aconst methods = deps.methods;initializePromise = runAsSystem(async () => {requireDeliveryOrigin(boundOrigin);+ purgeRecoveryLoop = createIdleRecoveryLoop({+ intervalMs: purgeRecoveryIntervalMs,+ maxIdleIntervalMs:+ options.idlePolling?.maintenanceMaxIntervalMs ??+ Math.max(purgeRecoveryIntervalMs, DEFAULT_PURGE_RECOVERY_MAX_IDLE_INTERVAL_MS),+ scan: recoverPurges,+ onError: (error) =>+ logger.error('[agent-triggers] durable delivery maintenance failed:', error),+ });await methods.ensureAgentTriggerDeliveryIndexes();if (stopping || isShutdownInProgress()) {throw new AgentTriggerServiceUnavailableError(@@ -514,13 +587,20 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}deliveryEngine = createAgentTriggerDeliveryEngine({- store: createDeliveryStore(methods, supportsDetachedActionCompletion),+ store: createDeliveryStore(methods, supportsDetachedActionCompletion, () =>+ purgeRecoveryLoop?.wake(),+ ),dispatch: dispatchForActivePrincipal,...(deps.settleSourceBeforeDeadLetter != null && {settleSourceBeforeDeadLetter: deps.settleSourceBeforeDeadLetter,}),},- deps.deliveryOptions,+ {+ ...deps.deliveryOptions,+ ...(options.idlePolling?.deliveryMaxIntervalMs != null && {+ maxIdleTickMs: options.idlePolling.deliveryMaxIntervalMs,+ }),+ },);deliveryReady = true;deliveryEngine.start();@@ -546,7 +626,10 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A};await requireActivePrincipal(String(prepared.user));const queued = await runAsSystem(async () =>- methods.enqueueAgentTriggerDelivery(durableDelivery),+ withMaintenanceRecovery(+ () => methods.enqueueAgentTriggerDelivery(durableDelivery),+ () => purgeRecoveryLoop?.wake(),+ ),);try {await requireActivePrincipal(String(prepared.user));@@ -589,7 +672,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): ArunAsSystem(async () => requireMethods().getAgentTriggerDeadLetters(limit)),requeue: (id, availableAt = new Date()) =>runAsSystem(async () => {- const revived = await requireMethods().requeueAgentTriggerDelivery(id, availableAt);+ const methods = requireMethods();+ const revived = await withMaintenanceRecovery(+ () => methods.requeueAgentTriggerDelivery(id, availableAt),+ () => purgeRecoveryLoop?.wake(),+ );if (revived != null) {if (availableAt.getTime() > Date.now()) {deliveryEngine?.noteEligibleAt(availableAt);@@ -601,14 +688,22 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}),retire: (deliveryKey, sourceId, reason, options) =>runAsSystem(async () => {- const retired = await requireCleanupMethods().retireAgentTriggerDelivery({- deliveryKey,- sourceId,- reason,- settledAt: new Date(),- ...(options?.onlyIfUnclaimed === true ? { onlyIfUnclaimed: true } : {}),- ...(options?.onlyIfDead === true ? { onlyIfDead: true } : {}),- });+ const methods = requireCleanupMethods();+ const retired = await withMaintenanceRecovery(+ (recovery) =>+ methods.retireAgentTriggerDelivery(+ {+ deliveryKey,+ sourceId,+ reason,+ settledAt: new Date(),+ ...(options?.onlyIfUnclaimed === true ? { onlyIfUnclaimed: true } : {}),+ ...(options?.onlyIfDead === true ? { onlyIfDead: true } : {}),+ },+ recovery,+ ),+ () => purgeRecoveryLoop?.wake(),+ );if (retired) {deliveryEngine?.wake();}@@ -640,9 +735,14 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}),drainUser,prepareUserPurge: (userId, fenceStartedAt, tenantId) =>- runAsSystem(async () =>- requireCleanupMethods().prepareAgentTriggerUserPurge(userId, fenceStartedAt, tenantId),- ),+ runAsSystem(async () => {+ const methods = requireCleanupMethods();+ await withMaintenanceRecovery(+ () => methods.prepareAgentTriggerUserPurge(userId, fenceStartedAt, tenantId),+ () => purgeRecoveryLoop?.wake(),+ );+ purgeRecoveryLoop?.wake();+ }),cancelUserPurge: (userId, fenceStartedAt) =>runAsSystem(async () =>requireCleanupMethods().cancelAgentTriggerUserPurge(userId, fenceStartedAt),@@ -651,6 +751,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A// shutdown has begun. Persistence remains usable even though admissions// and the delivery engine are deliberately no longer ready.purgeUser: (userId) =>- runAsSystem(async () => requireCleanupMethods().deleteAgentTriggerDeliveriesByUser(userId)),+ runAsSystem(async () => {+ const methods = requireCleanupMethods();+ await withMaintenanceRecovery(+ () => methods.deleteAgentTriggerDeliveriesByUser(userId),+ () => purgeRecoveryLoop?.wake(),+ );+ purgeRecoveryLoop?.wake();+ }),};}diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.tsindex d41e03d260..02676ab8ad 100644--- a/packages/data-provider/src/config.spec.ts+++ b/packages/data-provider/src/config.spec.ts@@ -673,6 +673,31 @@ describe('agent background completion batch config', () => {});describe('agent event runtime config', () => {+ it('defaults and bounds durable-worker idle polling intervals', () => {+ const parsed = configSchema.parse({+ version: '1.0',+ endpoints: { agents: { eventDriven: { idlePolling: {} } } },+ });+ expect(parsed.endpoints?.agents?.eventDriven?.idlePolling).toEqual({+ deliveryMaxIntervalMs: 15_000,+ queuedTurnMaxIntervalMs: 120_000,+ maintenanceMaxIntervalMs: 120_000,+ });+ for (const [key, value] of [+ ['deliveryMaxIntervalMs', 0],+ ['queuedTurnMaxIntervalMs', 29_999],+ ['maintenanceMaxIntervalMs', 300_001],+ ['maintenanceMaxIntervalMs', 30_000.5],+ ] as const) {+ expect(+ configSchema.safeParse({+ version: '1.0',+ endpoints: { agents: { eventDriven: { idlePolling: { [key]: value } } } },+ }).success,+ ).toBe(false);+ }+ });+it('accepts the routing choice and ignores removed rollout fields', () => {const result = configSchema.safeParse({version: '1.0',diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 691b48ac1e..698a5a6406 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -1512,6 +1512,33 @@ export const agentsEndpointSchema = baseEndpointSchemaeventDriven: z.object({selfUrl: z.string().url().optional(),+ /** Every replica keeps polling Mongo as a crash-recovery fallback. Wakes+ * keep local delivery prompt; these caps bound work missed across replicas. */+ idlePolling: z+ .object({+ deliveryMaxIntervalMs: z+ .number()+ .int()+ .min(1_000)+ .max(300_000)+ .optional()+ .default(15_000),+ queuedTurnMaxIntervalMs: z+ .number()+ .int()+ .min(30_000)+ .max(300_000)+ .optional()+ .default(120_000),+ maintenanceMaxIntervalMs: z+ .number()+ .int()+ .min(30_000)+ .max(300_000)+ .optional()+ .default(120_000),+ })+ .optional(),}).optional(),/** Conversational background-task delivery policy. Automatic completion wakeups arediff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.tsindex 6c1b2eea3d..704327518d 100644--- a/packages/data-schemas/src/methods/conversation.spec.ts+++ b/packages/data-schemas/src/methods/conversation.spec.ts@@ -6449,7 +6449,9 @@ describe('Conversation Operations', () => {},]);- await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(1);+ const activity = { found: false };+ await expect(methods.expireLegacyAgentEventActorReceipts(now, 1, activity)).resolves.toBe(1);+ expect(activity.found).toBe(true);await expect(Conversation.findOne({ conversationId: oldConversationId }).select('+agentEventActorReconciliations')@@ -6625,9 +6627,20 @@ describe('Conversation Operations', () => {awaitTerminalHandling: true,});- /** The first raw page contains no expired receipt. The second contains- * a protected receipt, and only the third reaches removable work. */- await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(0);+ /** The first raw page contains no expired receipt. It must still keep+ * discovery active, without querying delivery protection for an empty ID set.+ * The second is protected, and only the third reaches removable work. */+ const activity = { found: false };+ const protection = jest.spyOn(Delivery, 'find');+ try {+ await expect(methods.expireLegacyAgentEventActorReceipts(now, 1, activity)).resolves.toBe(+ 0,+ );+ expect(activity.found).toBe(true);+ expect(protection).not.toHaveBeenCalled();+ } finally {+ protection.mockRestore();+ }await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(0);await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(1);await expect(diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.tsindex f81e52fb11..71811f2f44 100644--- a/packages/data-schemas/src/methods/conversation.ts+++ b/packages/data-schemas/src/methods/conversation.ts@@ -449,7 +449,11 @@ export interface ConversationMethods {getAgentEventActorReconciliationStorageMetrics(now: Date,): Promise<AgentEventActorReconciliationStorageMetrics>;- expireLegacyAgentEventActorReceipts(now: Date, limit?: number): Promise<number>;+ expireLegacyAgentEventActorReceipts(+ now: Date,+ limit?: number,+ activity?: { found: boolean },+ ): Promise<number>;reserveSubagentThread(input: {user: string;conversationId: string;@@ -1782,7 +1786,11 @@ export function createConversationMethods(/** Bounded mixed-version cleanup for terminal receipts embedded by older* builds. New receipts expire through the delivery collection TTL index,* but dormant legacy conversations need an independent retirement path. */- async function expireLegacyAgentEventActorReceipts(now: Date, limit = 100): Promise<number> {+ async function expireLegacyAgentEventActorReceipts(+ now: Date,+ limit = 100,+ activity?: { found: boolean },+ ): Promise<number> {if (Number.isNaN(now.getTime())) {throw new TypeError('now must be a valid date');}@@ -1821,6 +1829,9 @@ export function createConversationMethods(return 0;}legacyReceiptExpiryCursor = candidates[candidates.length - 1]._id;+ // A full page is not an empty sweep: later conversations may contain+ // expired receipts. Keep scanning at base cadence until the page walk ends.+ if (activity != null && candidates.length === boundedLimit) activity.found = true;const expiredInvocationIds = [...new Set(candidates.flatMap((candidate) =>@@ -1830,6 +1841,8 @@ export function createConversationMethods(),),];+ if (expiredInvocationIds.length === 0) return 0;+ if (activity != null) activity.found = true;const protectedInvocationIds = new Set(await Delivery.find({deliveryKey: { $in: expiredInvocationIds },diff --git a/packages/data-schemas/src/methods/queuedTurn.spec.ts b/packages/data-schemas/src/methods/queuedTurn.spec.tsindex f4718dca8e..74c2f95f54 100644--- a/packages/data-schemas/src/methods/queuedTurn.spec.ts+++ b/packages/data-schemas/src/methods/queuedTurn.spec.ts@@ -220,6 +220,31 @@ describe('agent queued turn methods', () => {).resolves.toMatchObject({ status: 'queued', sequence: expect.any(Number) });});+ it('reports reservation-only recovery activity when deleted lanes yield no deliveries', async () => {+ const first = await methods.enqueueAgentQueuedTurn(+ enqueueInput({ clientRequestId: 'lost-lane-first' }),+ );+ const second = await methods.enqueueAgentQueuedTurn(+ enqueueInput({ clientRequestId: 'lost-lane-second' }),+ );+ await Turn.updateMany(+ { _id: { $in: [first.turn.queuedTurnId, second.turn.queuedTurnId] } },+ {+ $set: { status: 'reserving' },+ $unset: { sequence: 1 },+ },+ );+ await Sequence.deleteMany({});+ for (let i = 0; i < 2; i++) {+ const activity = { found: false };+ expect(await methods.findQueuedTurnsNeedingDelivery(1, activity)).toEqual([]);+ expect(activity.found).toBe(true);+ }+ const empty = { found: false };+ expect(await methods.findQueuedTurnsNeedingDelivery(1, empty)).toEqual([]);+ expect(empty.found).toBe(false);+ });+it('caps each conversation at 100 active turns while preserving exact replay', async () => {const inputs = Array.from({ length: 101 }, (_, index) =>enqueueInput({ clientRequestId: `capacity-${index}`, text: `turn ${index}` }),diff --git a/packages/data-schemas/src/methods/queuedTurn.ts b/packages/data-schemas/src/methods/queuedTurn.tsindex 22085ba0d9..f10bcb17ae 100644--- a/packages/data-schemas/src/methods/queuedTurn.ts+++ b/packages/data-schemas/src/methods/queuedTurn.ts@@ -203,9 +203,13 @@ export interface AgentQueuedTurnMethods {listAgentQueuedTurnReceipts: (input: AgentQueuedTurnConversationScope & { clientRequestIds?: readonly string[] },) => Promise<AgentQueuedTurnActiveRecord[]>;- findQueuedTurnsNeedingDelivery: (limit?: number) => Promise<AgentQueuedTurnRecord[]>;+ findQueuedTurnsNeedingDelivery: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<AgentQueuedTurnRecord[]>;claimQueuedTurnsForAdmissionReconciliation: (input: ClaimAgentQueuedTurnReconciliationInput,+ activity?: { found: boolean },) => Promise<AgentQueuedTurnRecord[]>;deferAgentQueuedTurnAdmissionReconciliation: (input: AgentQueuedTurnConversationScope & {@@ -1381,7 +1385,10 @@ export function createAgentQueuedTurnMethods(return { outcome: 'not_cancellable', turn: toRecord(current) };}- async function findQueuedTurnsNeedingDelivery(limit = 100): Promise<AgentQueuedTurnRecord[]> {+ async function findQueuedTurnsNeedingDelivery(+ limit = 100,+ activity?: { found: boolean },+ ): Promise<AgentQueuedTurnRecord[]> {if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1000) {throw new TypeError('Agent queued turn recovery limit must be between 1 and 1000');}@@ -1390,6 +1397,7 @@ export function createAgentQueuedTurnMethods(.sort({ createdAt: 1, _id: 1 }).limit(limit).lean<IAgentQueuedTurn[]>();+ if (activity != null && reservations.length > 0) activity.found = true;for (const reservation of reservations) {const scopeInput: AgentQueuedTurnConversationScope = {user: reservation.user,@@ -1460,6 +1468,7 @@ export function createAgentQueuedTurnMethods(async function claimQueuedTurnsForAdmissionReconciliation(input: ClaimAgentQueuedTurnReconciliationInput,+ activity?: { found: boolean },): Promise<AgentQueuedTurnRecord[]> {const limit = input.limit ?? 100;if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1000) {@@ -1519,6 +1528,7 @@ export function createAgentQueuedTurnMethods(if (candidates.length === 0) {return [];}+ if (activity != null) activity.found = true;await Turn().updateMany({...eligible,diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.tsindex 1f0fcc74f6..c2b6110930 100644--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts@@ -1744,6 +1744,74 @@ describe('agent trigger delivery methods', () => {});});+ it.each(['complete', 'dead', 'cleanup'] as const)(+ 'signals maintenance after %s finalization fails without undoing root settlement',+ async (mode) => {+ const user = new mongoose.Types.ObjectId();+ const coalesceUntil = new Date(Date.now() + 60_000);+ const shared = {+ user,+ orderingKey: 'inline-recovery',+ coalesceKey: 'inline-batch',+ coalesceFrom: new Date(coalesceUntil.getTime() - 750),+ coalesceUntil,+ availableAt: coalesceUntil,+ envelopeBytes: 128,+ };+ const root = await methods.enqueueAgentTriggerDelivery(enqueueInput(shared));+ if (mode !== 'cleanup') await methods.enqueueAgentTriggerDelivery(enqueueInput(shared));+ const claimed = await methods.claimNextAgentTriggerDelivery({+ workerId: 'worker-1',+ claimToken: 'inline-finalization-claim',+ now: coalesceUntil,+ leaseUntil: new Date(coalesceUntil.getTime() + 60_000),+ });+ expect(claimed?.id).toBe(root.delivery.id);+ const input = {+ id: claimed!.id,+ workerId: 'worker-1',+ claimToken: claimed!.claimToken!,+ attempt: 1,+ settledAt: coalesceUntil,+ };+ const failure =+ mode === 'cleanup'+ ? jest+ .spyOn(LaneSequence, 'updateOne')+ .mockRejectedValueOnce(new Error('cleanup interrupted'))+ : jest+ .spyOn(Delivery, 'updateMany')+ .mockRejectedValueOnce(new Error('batch interrupted'));+ const recovery = { required: false };+ try {+ const settled =+ mode === 'dead'+ ? await methods.deadLetterAgentTriggerDelivery(+ { ...input, error: transientFailure({ retryable: false }) },+ recovery,+ )+ : await methods.completeAgentTriggerDelivery(+ { ...input, result: { accepted: true } },+ recovery,+ );+ expect(settled).toBe(true);+ expect(recovery.required).toBe(true);+ } finally {+ failure.mockRestore();+ }+ expect((await Delivery.findById(root.delivery.id).lean())?.status).toBe(+ mode === 'dead' ? 'dead' : 'succeeded',+ );+ await methods.recoverAgentTriggerBatchReceipts();+ await methods.reclaimInactiveAgentTriggerLanes();+ const rows = await Delivery.find({ orderingKey: shared.orderingKey }).lean();+ expect(rows.every((row) => row.status === (mode === 'dead' ? 'dead' : 'succeeded'))).toBe(+ true,+ );+ if (mode !== 'dead') expect(await LaneSequence.findById(shared.orderingKey)).toBeNull();+ },+ );+it('recovers batch receipts and lane cleanup after root settlement was interrupted', async () => {const user = new mongoose.Types.ObjectId();const coalesceUntil = new Date(Date.now() + 60_000);@@ -2244,6 +2312,9 @@ describe('agent trigger delivery methods', () => {});it('leaves staging unpublished while its durable user purge marker exists', async () => {+ const emptyActivity = { found: false };+ await expect(methods.recoverAgentTriggerLanePublications(1, emptyActivity)).resolves.toBe(0);+ expect(emptyActivity.found).toBe(false);const user = new mongoose.Types.ObjectId();const orderingKey = 'purge-fenced-staging';await UserPurge.create({ _id: user, fenceStartedAt: START, tenantId: 'tenant-1' });@@ -2256,7 +2327,9 @@ describe('agent trigger delivery methods', () => {stagingRecoveryAt: START,});- await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);+ const activity = { found: false };+ await expect(methods.recoverAgentTriggerLanePublications(1, activity)).resolves.toBe(0);+ expect(activity.found).toBe(true);await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({status: 'staging',laneSequence: 0,diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.tsindex 0137a6dd43..e3fe4c2d0f 100644--- a/packages/data-schemas/src/methods/triggerDelivery.ts+++ b/packages/data-schemas/src/methods/triggerDelivery.ts@@ -281,18 +281,22 @@ export interface AgentTriggerDeliveryMethods {handling?: AgentTriggerHandlingState;awaitTerminalHandling?: true;},+ recovery?: { required: boolean },+ ) => Promise<boolean>;+ retireAgentTriggerDelivery: (+ input: {+ deliveryKey: string;+ sourceId: string;+ settledAt: Date;+ reason: string;+ onlyIfUnclaimed?: boolean;+ onlyIfDead?: boolean;+ /** Accept transport success without a terminal handling receipt, unless the+ * delivery explicitly keeps its lane open for terminal handling. */+ allowSucceeded?: boolean;+ },+ recovery?: { required: boolean },) => Promise<boolean>;- retireAgentTriggerDelivery: (input: {- deliveryKey: string;- sourceId: string;- settledAt: Date;- reason: string;- onlyIfUnclaimed?: boolean;- onlyIfDead?: boolean;- /** Accept transport success without a terminal handling receipt, unless the- * delivery explicitly keeps its lane open for terminal handling. */- allowSucceeded?: boolean;- }) => Promise<boolean>;renewAgentTriggerDeliveryProducerLease: (input: {deliveryKey: string;sourceId: string;@@ -385,6 +389,7 @@ export interface AgentTriggerDeliveryMethods {settledAt: Date;receiptRetryAt?: Date;},+ recovery?: { required: boolean },) => Promise<boolean>;getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;getAgentTriggerDeliveryStatus: (@@ -402,9 +407,18 @@ export interface AgentTriggerDeliveryMethods {user: string | Types.ObjectId,now: Date,) => Promise<number>;- recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;- recoverAgentTriggerBatchReceipts: (limit?: number) => Promise<number>;- reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;+ recoverAgentTriggerLanePublications: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;+ recoverAgentTriggerBatchReceipts: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;+ reclaimInactiveAgentTriggerLanes: (+ limit?: number,+ activity?: { found: boolean },+ ) => Promise<number>;prepareAgentTriggerUserPurge: (user: string | Types.ObjectId,fenceStartedAt: Date,@@ -414,7 +428,7 @@ export interface AgentTriggerDeliveryMethods {user: string | Types.ObjectId,fenceStartedAt: Date,) => Promise<boolean>;- recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;+ recoverAgentTriggerUserPurges: (limit?: number, activity?: { found: boolean }) => Promise<number>;deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;eraseAgentTriggerDeliveryConversationResults: (user: string | Types.ObjectId,@@ -1092,6 +1106,7 @@ export function createAgentTriggerDeliveryMethods(/** Repairs abandoned reservations and staging rows left by crashed writers. */async function recoverAgentTriggerLanePublications(limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ activity?: { found: boolean },): Promise<number> {if (!Number.isSafeInteger(limit) || limit <= 0) {throw new TypeError('Agent trigger lane recovery limit must be a positive integer');@@ -1105,6 +1120,7 @@ export function createAgentTriggerDeliveryMethods(.sort({ publisherStartedAt: 1, _id: 1 }).limit(boundedLimit).lean<IAgentTriggerLaneSequence[]>();+ if (activity != null && lanes.length > 0) activity.found = true;let recovered = 0;const recoveryCursor = new Date();for (const lane of lanes) {@@ -1175,6 +1191,9 @@ export function createAgentTriggerDeliveryMethods(.limit(remaining).lean<IAgentTriggerDelivery[]>(): [];+ if (activity != null && (legacyStaged.length > 0 || indexedStaged.length > 0)) {+ activity.found = true;+ }const staged = [...legacyStaged.map((delivery) => ({...delivery,@@ -1293,6 +1312,7 @@ export function createAgentTriggerDeliveryMethods(/** Bounds high-cardinality ordering metadata after the final retained job settles. */async function reclaimInactiveAgentTriggerLanes(limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ activity?: { found: boolean },): Promise<number> {if (!Number.isSafeInteger(limit) || limit <= 0) {throw new TypeError('Agent trigger lane reclamation limit must be a positive integer');@@ -1304,6 +1324,7 @@ export function createAgentTriggerDeliveryMethods(.limit(boundedLimit).select('_id orderingKey laneCleanupPendingAt').lean<Array<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>>();+ if (activity != null && pendingCleanup.length > 0) activity.found = true;let reclaimed = 0;for (const delivery of pendingCleanup) {if (await fulfillLaneCleanupRequest(delivery)) {@@ -1321,6 +1342,7 @@ export function createAgentTriggerDeliveryMethods(.limit(remaining).select('_id').lean<Array<Pick<IAgentTriggerLaneSequence, '_id'>>>();+ if (activity != null && lanes.length > 0) activity.found = true;for (const lane of lanes) {if (await reclaimLaneIfInactive(lane._id)) {reclaimed += 1;@@ -1760,6 +1782,7 @@ export function createAgentTriggerDeliveryMethods(async function recoverAgentTriggerBatchReceipts(limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ activity?: { found: boolean },): Promise<number> {if (!Number.isSafeInteger(limit) || limit <= 0) {throw new TypeError('Agent trigger batch recovery limit must be a positive integer');@@ -1773,6 +1796,7 @@ export function createAgentTriggerDeliveryMethods(.sort({ settledAt: 1, _id: 1 }).limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT)).lean<IAgentTriggerDelivery[]>();+ if (activity != null && roots.length > 0) activity.found = true;let recovered = 0;for (const root of roots) {if (root.settledAt == null || (root.status !== 'succeeded' && root.status !== 'dead')) {@@ -1970,6 +1994,7 @@ export function createAgentTriggerDeliveryMethods(handling?: AgentTriggerHandlingState;awaitTerminalHandling?: true;},+ recovery?: { required: boolean },): Promise<boolean> {const awaitsTerminalHandling =input.awaitTerminalHandling === true && input.handling?.status === 'started';@@ -2050,6 +2075,7 @@ export function createAgentTriggerDeliveryMethods(});await fulfillLaneCleanupRequest(completed);} catch (error) {+ if (recovery != null) recovery.required = true;// Root success is authoritative. Maintenance retries both constituent// receipt settlement and the existing durable lane-cleanup marker.logger.warn('[agent-triggers] failed to finalize a completed trigger batch', {@@ -2064,17 +2090,20 @@ export function createAgentTriggerDeliveryMethods(* that the result will never become dispatchable. This transition is keyed* by the immutable delivery identity rather than a worker lease so the* producer can unblock the lane even while a resolver is deferring it. */- async function retireAgentTriggerDelivery(input: {- deliveryKey: string;- sourceId: string;- settledAt: Date;- reason: string;- onlyIfUnclaimed?: boolean;- onlyIfDead?: boolean;- /** Accept transport success without a terminal handling receipt, unless the- * delivery explicitly keeps its lane open for terminal handling. */- allowSucceeded?: boolean;- }): Promise<boolean> {+ async function retireAgentTriggerDelivery(+ input: {+ deliveryKey: string;+ sourceId: string;+ settledAt: Date;+ reason: string;+ onlyIfUnclaimed?: boolean;+ onlyIfDead?: boolean;+ /** Accept transport success without a terminal handling receipt, unless the+ * delivery explicitly keeps its lane open for terminal handling. */+ allowSucceeded?: boolean;+ },+ recovery?: { required: boolean },+ ): Promise<boolean> {if (input.deliveryKey.length === 0 ||input.deliveryKey.length > 256 ||@@ -2151,6 +2180,7 @@ export function createAgentTriggerDeliveryMethods(try {await fulfillLaneCleanupRequest(retired);} catch (error) {+ if (recovery != null) recovery.required = true;logger.warn('[agent-triggers] failed to finalize a retired internal delivery', {deliveryKey: input.deliveryKey,error: error instanceof Error ? error.message : String(error),@@ -3509,6 +3539,7 @@ export function createAgentTriggerDeliveryMethods(settledAt: Date;receiptRetryAt?: Date;},+ recovery?: { required: boolean },): Promise<boolean> {const error = normalizeFailure(input.error);if (@@ -3603,6 +3634,7 @@ export function createAgentTriggerDeliveryMethods(error,});} catch (settlementError) {+ if (recovery != null) recovery.required = true;logger.warn('[agent-triggers] failed to settle batch dead-letter receipts', {deliveryId: String(dead._id),error: settlementError instanceof Error ? settlementError.message : String(settlementError),@@ -3881,6 +3913,7 @@ export function createAgentTriggerDeliveryMethods(/** Recovers cleanup markers whose users are gone; active-user markers are never destructive. */async function recoverAgentTriggerUserPurges(limit = DEFAULT_PURGE_RECOVERY_LIMIT,+ activity?: { found: boolean },): Promise<number> {if (!Number.isSafeInteger(limit) || limit <= 0) {throw new TypeError('Agent trigger purge recovery limit must be a positive integer');@@ -3893,6 +3926,7 @@ export function createAgentTriggerDeliveryMethods(.sort({ backgroundToolResultDeletionPendingAt: 1, _id: 1 }).limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT)).lean<IAgentTriggerDelivery[]>();+ if (activity != null && pending.length > 0) activity.found = true;for (const row of pending) {const conversationId = (row.envelope as { target?: { conversationId?: string } }).target?.conversationId;@@ -3915,6 +3949,7 @@ export function createAgentTriggerDeliveryMethods(.sort({ updatedAt: 1, _id: 1 }).limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT)).lean<IAgentTriggerUserPurge[]>();+ if (activity != null && markers.length > 0) activity.found = true;let recovered = 0;for (const marker of markers) {const user = await mongoose.models.User.findById(marker._id)