Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔕 perf: Back Off Idle Durable Agent Recovery Polls (#16303)

lia-by-librechat[bot]ffd3d1e139ce
diff --git a/api/server/experimental.js b/api/server/experimental.js
index d6b610a595..45332738d4 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -709,6 +709,7 @@ if (cluster.isMaster) {
address: server.address(),
completionResultBatchSize:
baseAppConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
+ idlePolling: baseAppConfig?.endpoints?.agents?.eventDriven?.idlePolling,
});
} catch (initErr) {
logger.error(`Worker ${process.pid} post-listen initialization failed:`, initErr);
diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.js
index abe310b1ec..b1f05d4663 100644
--- a/api/server/experimental.spec.js
+++ b/api/server/experimental.spec.js
@@ -95,6 +95,19 @@ describe('Experimental server configuration', () => {
expect(listenIndex).toBeGreaterThan(eventRuntimeIndex);
});
+ it('passes the same idle recovery policy to both server startup paths', () => {
+ const standard = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8');
+ for (const [entrypoint, config] of [
+ [source, 'baseAppConfig'],
+ [standard, 'appConfig'],
+ ]) {
+ const start = entrypoint.indexOf('await initializeAgentTriggerService({');
+ expect(start).toBeGreaterThan(-1);
+ const call = entrypoint.slice(start, entrypoint.indexOf('});', start));
+ expect(call).toContain(`idlePolling: ${config}?.endpoints?.agents?.eventDriven?.idlePolling`);
+ }
+ });
+
it('matches the standard server pre-authentication tenant routes', () => {
expect(source).toContain("app.use('/oauth', preAuthTenantMiddleware, routes.oauth);");
expect(source).toContain("app.use('/api/auth', preAuthTenantMiddleware, routes.auth);");
diff --git a/api/server/index.js b/api/server/index.js
index b4555e7b26..1d31f63b26 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -502,6 +502,7 @@ const startServer = async () => {
address: server.address(),
completionResultBatchSize:
appConfig?.endpoints?.agents?.backgroundTasks?.completionResultBatchSize,
+ idlePolling: appConfig?.endpoints?.agents?.eventDriven?.idlePolling,
});
const scheduleEngineArmed = (await initializeScheduleEngine()) != null;
scheduleEngineState = scheduleEngineArmed ? 'armed' : 'unavailable';
diff --git a/api/server/services/Agents/triggers.js b/api/server/services/Agents/triggers.js
index 0f2d81218e..7b58b54a9e 100644
--- a/api/server/services/Agents/triggers.js
+++ b/api/server/services/Agents/triggers.js
@@ -64,7 +64,9 @@ service = createAgentTriggerService({
const initializeAgentTriggerService = async (options) => {
await service.initialize(options);
- await queuedTurnLifecycle.initialize();
+ await queuedTurnLifecycle.initialize({
+ maxIdleIntervalMs: options?.idlePolling?.queuedTurnMaxIntervalMs,
+ });
};
const stopAgentTriggerService = async () => {
diff --git a/api/server/services/Agents/triggers.spec.js b/api/server/services/Agents/triggers.spec.js
index 4c636f4692..df3a09b2d6 100644
--- a/api/server/services/Agents/triggers.spec.js
+++ b/api/server/services/Agents/triggers.spec.js
@@ -80,6 +80,23 @@ describe('agent trigger service composition', () => {
expect(supportsDetachedActionCompletion()).toBe(false);
});
+ it('passes the recovery policy to both delivery and queued-turn workers', async () => {
+ const { initializeAgentTriggerService } = require('./triggers');
+ const options = {
+ address: 'local',
+ idlePolling: {
+ queuedTurnMaxIntervalMs: 60_000,
+ maintenanceMaxIntervalMs: 90_000,
+ deliveryMaxIntervalMs: 5_000,
+ },
+ };
+ await initializeAgentTriggerService(options);
+ expect(mockCreateAgentTriggerService.mock.results[0].value.initialize).toHaveBeenCalledWith(
+ options,
+ );
+ expect(mockQueuedTurnLifecycle.initialize).toHaveBeenCalledWith({ maxIdleIntervalMs: 60_000 });
+ });
+
it('injects the configured background completion batch size', async () => {
const { initializeAgentTriggerService } = require('./triggers');
await initializeAgentTriggerService({ address: 'local', completionResultBatchSize: 12 });
diff --git a/packages/api/src/agents/checkpoints/deletion.integration.spec.ts b/packages/api/src/agents/checkpoints/deletion.integration.spec.ts
index 84d8b59eb9..5da8e089e3 100644
--- a/packages/api/src/agents/checkpoints/deletion.integration.spec.ts
+++ b/packages/api/src/agents/checkpoints/deletion.integration.spec.ts
@@ -242,7 +242,9 @@ test('retains writer obligations, renewed revisions, and lookup failures', async
await deletion.remember(['child']);
const getJobs = jest.fn().mockResolvedValue(['pending-host']);
const reclaim = createCheckpointDeletionReclaimer(getJobs);
- expect(await reclaim(25)).toBe(0);
+ const activity = { found: false };
+ expect(await reclaim(25, activity)).toBe(0);
+ expect(activity.found).toBe(true);
getJobs.mockRejectedValueOnce(new Error('job store unavailable'));
await expect(reclaim(25)).rejects.toThrow('reclamation failed');
getJobs.mockImplementationOnce(async () => {
diff --git a/packages/api/src/agents/checkpoints/deletion.ts b/packages/api/src/agents/checkpoints/deletion.ts
index 074341dc2f..36fcd1a0cc 100644
--- a/packages/api/src/agents/checkpoints/deletion.ts
+++ b/packages/api/src/agents/checkpoints/deletion.ts
@@ -225,9 +225,9 @@ export async function openCheckpointDeletion(
/** Replay captured identities after topology deletion; never sweep new generations. */
export function createCheckpointDeletionReclaimer(
getOwnerJobs: (userId: string, tenantId?: string) => Promise<string[]>,
-): (limit: number) => Promise<number> {
+): (limit: number, activity?: { found: boolean }) => Promise<number> {
let after: string | undefined;
- return async (limit) => {
+ return async (limit, activity) => {
if (!Number.isSafeInteger(limit) || limit <= 0) throw new Error('Invalid reclamation limit');
const db = mongoose.connection.db;
if (!db || mongoose.connection.readyState !== 1)
@@ -238,6 +238,7 @@ export function createCheckpointDeletionReclaimer(
.sort({ _id: 1 })
.limit(limit)
.toArray();
+ if (activity != null && targets.length > 0) activity.found = true;
after = targets.length === limit ? targets[targets.length - 1]._id : undefined;
const jobsByOwner = new Map<string, Promise<string[]>>();
diff --git a/packages/api/src/agents/queuedTurns.spec.ts b/packages/api/src/agents/queuedTurns.spec.ts
index adb41ec79d..b4157fd837 100644
--- a/packages/api/src/agents/queuedTurns.spec.ts
+++ b/packages/api/src/agents/queuedTurns.spec.ts
@@ -843,6 +843,199 @@ describe('Agent queued-turn continuation', () => {
});
describe('Agent queued-turn delivery scheduling', () => {
+ it('can retry initialization after a transient index setup failure', async () => {
+ const ensureAgentQueuedTurnIndexes = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('mongo unavailable'))
+ .mockResolvedValue(undefined);
+ const scheduler = createAgentQueuedTurnScheduler({
+ methods: {
+ ensureAgentQueuedTurnIndexes,
+ findQueuedTurnsNeedingDelivery: jest.fn(async () => []),
+ claimQueuedTurnsForAdmissionReconciliation: jest.fn(async () => []),
+ } as unknown as AgentQueuedTurnMethods,
+ enqueue: jest.fn(),
+ getGenerationAdmissionEvidence: async () => null,
+ });
+ await expect(scheduler.initialize()).rejects.toThrow('mongo unavailable');
+ await expect(scheduler.initialize()).resolves.toBeUndefined();
+ expect(ensureAgentQueuedTurnIndexes).toHaveBeenCalledTimes(2);
+ await scheduler.stop();
+ });
+
+ it('does not back off when discovery repairs reservations but returns no deliveries', async () => {
+ jest.useFakeTimers();
+ const findQueuedTurnsNeedingDelivery = jest.fn(
+ async (_limit: number, activity: { found: boolean }) => {
+ activity.found = true;
+ return [];
+ },
+ );
+ const scheduler = createAgentQueuedTurnScheduler({
+ methods: {
+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),
+ findQueuedTurnsNeedingDelivery,
+ claimQueuedTurnsForAdmissionReconciliation: jest.fn(async () => []),
+ } as unknown as AgentQueuedTurnMethods,
+ enqueue: jest.fn(),
+ getGenerationAdmissionEvidence: async () => null,
+ });
+ try {
+ await scheduler.initialize();
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(2);
+ } finally {
+ await scheduler.stop();
+ jest.useRealTimers();
+ }
+ });
+
+ it('waits for sibling lease discovery after an error before allowing shutdown or another scan', async () => {
+ jest.useFakeTimers();
+ let release!: (turns: AgentQueuedTurnRecord[]) => void;
+ const findQueuedTurnsNeedingDelivery = jest
+ .fn()
+ .mockRejectedValue(new Error('mongo unavailable'));
+ const claimQueuedTurnsForAdmissionReconciliation = jest.fn(
+ () =>
+ new Promise<AgentQueuedTurnRecord[]>((resolve) => {
+ release = resolve;
+ }),
+ );
+ const scheduler = createAgentQueuedTurnScheduler({
+ methods: {
+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),
+ findQueuedTurnsNeedingDelivery,
+ claimQueuedTurnsForAdmissionReconciliation,
+ } as unknown as AgentQueuedTurnMethods,
+ enqueue: jest.fn(),
+ getGenerationAdmissionEvidence: async () => null,
+ });
+ try {
+ const starting = scheduler.initialize();
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(1);
+ let stopped = false;
+ const stopping = scheduler.stop().then(() => {
+ stopped = true;
+ });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(stopped).toBe(false);
+ release([]);
+ await Promise.all([starting, stopping]);
+ expect(stopped).toBe(true);
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(1);
+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(1);
+ } finally {
+ await scheduler.stop();
+ jest.useRealTimers();
+ }
+ });
+
+ it('reduces empty Mongo discovery reads without losing bounded fallback polling', async () => {
+ jest.useFakeTimers();
+ try {
+ const findQueuedTurnsNeedingDelivery = jest.fn(async () => []);
+ const claimQueuedTurnsForAdmissionReconciliation = jest.fn(async () => []);
+ const scheduler = createAgentQueuedTurnScheduler({
+ methods: {
+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),
+ findQueuedTurnsNeedingDelivery,
+ claimQueuedTurnsForAdmissionReconciliation,
+ } as unknown as AgentQueuedTurnMethods,
+ enqueue: jest.fn(),
+ getGenerationAdmissionEvidence: async () => null,
+ });
+ await scheduler.initialize();
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(60_000);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(3);
+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(3);
+ await scheduler.stop();
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(3);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('does not treat a deferred indeterminate admission as an empty queue', async () => {
+ jest.useFakeTimers();
+ jest.setSystemTime(new Date(NOW));
+ try {
+ const turn = {
+ ...queuedTurn('queued-turn-deferred', 1),
+ status: 'dead' as const,
+ deliveryKey: 'delivery-deferred',
+ admissionId: 'delivery-deferred',
+ admissionProtocolVersion: 2 as const,
+ terminalReceipt: {
+ outcome: 'dead' as const,
+ settledAt: new Date(NOW),
+ failure: { code: 'ADMISSION_INDETERMINATE', message: 'pending evidence' },
+ },
+ };
+ const claimQueuedTurnsForAdmissionReconciliation = jest.fn(async (input) => [
+ {
+ ...turn,
+ reconciliationClaimId: input.claimId,
+ reconciliationClaimBy: input.claimBy,
+ },
+ ]);
+ const deferAgentQueuedTurnAdmissionReconciliation = jest.fn(async () => true);
+ const scheduler = createAgentQueuedTurnScheduler({
+ methods: {
+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),
+ findQueuedTurnsNeedingDelivery: jest.fn(async () => []),
+ claimQueuedTurnsForAdmissionReconciliation,
+ deferAgentQueuedTurnAdmissionReconciliation,
+ } as unknown as AgentQueuedTurnMethods,
+ enqueue: jest.fn(),
+ getGenerationAdmissionEvidence: async () => null,
+ });
+ await scheduler.initialize();
+ expect(deferAgentQueuedTurnAdmissionReconciliation).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(4_999);
+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(1);
+ expect(claimQueuedTurnsForAdmissionReconciliation).toHaveBeenCalledTimes(2);
+ await scheduler.stop();
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('wakes recovery after direct publication fails outside its own recovery pass', async () => {
+ jest.useFakeTimers();
+ try {
+ const findQueuedTurnsNeedingDelivery = jest.fn(async () => []);
+ const scheduler = createAgentQueuedTurnScheduler({
+ methods: {
+ ensureAgentQueuedTurnIndexes: jest.fn(async () => undefined),
+ findQueuedTurnsNeedingDelivery,
+ claimQueuedTurnsForAdmissionReconciliation: jest.fn(async () => []),
+ reserveAgentQueuedTurnDelivery: jest.fn(async () => {
+ throw new Error('mongo unavailable');
+ }),
+ } as unknown as AgentQueuedTurnMethods,
+ enqueue: jest.fn(),
+ getGenerationAdmissionEvidence: async () => null,
+ });
+ await scheduler.initialize();
+ await expect(scheduler.schedule(queuedTurn('queued-turn-1', 1))).rejects.toThrow(
+ 'mongo unavailable',
+ );
+ await jest.advanceTimersByTimeAsync(0);
+ expect(findQueuedTurnsNeedingDelivery).toHaveBeenCalledTimes(2);
+ await scheduler.stop();
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
function queuedTurn(id: string, sequence: number): AgentQueuedTurnRecord {
const { claimId: _claimId, claimBy: _claimBy, claimUntil: _claimUntil, ...record } = claim();
return {
diff --git a/packages/api/src/agents/queuedTurns.ts b/packages/api/src/agents/queuedTurns.ts
index 6eeb6864f9..540e981b2d 100644
--- a/packages/api/src/agents/queuedTurns.ts
+++ b/packages/api/src/agents/queuedTurns.ts
@@ -23,6 +23,7 @@ import type { AgentTriggerDeliveryFailure } from './triggers/engine';
import { getAgentTriggerIdempotencyKey, parseAgentTriggerEnvelope } from './triggers/envelope';
import { createAgentTriggerEnvelope } from './triggers/envelope';
import { AgentTriggerExecutionError } from './triggers/host';
+import { createIdleRecoveryLoop } from './recovery';
export const AGENT_QUEUED_TURN_SOURCE = 'agent-queued-turn';
const AGENT_QUEUED_TURN_EVENT = 'agent.queued-turn';
@@ -32,6 +33,7 @@ const RECONCILIATION_LEASE_MS = 2 * 60 * 1000;
const RECONCILIATION_BACKOFF_BASE_MS = 5_000;
const RECONCILIATION_BACKOFF_MAX_MS = 5 * 60 * 1000;
const DEFAULT_RECOVERY_INTERVAL_MS = 30_000;
+const DEFAULT_RECOVERY_MAX_IDLE_INTERVAL_MS = 2 * 60_000;
const DEFAULT_RECOVERY_LIMIT = 100;
const MAX_FAILURE_CODE_LENGTH = 128;
const MAX_FAILURE_MESSAGE_LENGTH = 2048;
@@ -87,7 +89,7 @@ export interface AgentQueuedTurnSchedulerDeps {
}
export interface AgentQueuedTurnScheduler {
- initialize: () => Promise<void>;
+ initialize: (options?: { maxIdleIntervalMs?: number }) => Promise<void>;
stop: () => Promise<void>;
schedule: (turn: AgentQueuedTurnRecord) => Promise<string>;
recover: () => Promise<number>;
@@ -107,7 +109,7 @@ export interface AgentQueuedTurnLifecycle {
rawSource: unknown,
input: AgentQueuedTurnExecutionAdmission,
) => Promise<boolean>;
- initialize: () => Promise<void>;
+ initialize: (options?: { maxIdleIntervalMs?: number }) => Promise<void>;
stop: () => Promise<void>;
schedule: (turn: AgentQueuedTurnRecord) => Promise<string>;
cancel: (
@@ -732,10 +734,12 @@ function createAgentQueuedTurnScheduler({
recoveryIntervalMs = DEFAULT_RECOVERY_INTERVAL_MS,
recoveryLimit = DEFAULT_RECOVERY_LIMIT,
}: AgentQueuedTurnSchedulerDeps): AgentQueuedTurnScheduler {
- let timer: NodeJS.Timeout | undefined;
- let recovery: Promise<number> | undefined;
+ let loop: ReturnType<typeof createIdleRecoveryLoop> | undefined;
+ let initialization: Promise<void> | undefined;
+ let stopped = false;
+ let recovery: Promise<{ repaired: number; idle: boolean }> | undefined;
- const schedule = async (turn: AgentQueuedTurnRecord): Promise<string> => {
+ const publish = async (turn: AgentQueuedTurnRecord): Promise<string> => {
const envelope = deliveryEnvelope(turn);
const deliveryKey = getAgentTriggerIdempotencyKey(envelope);
const reserved = await runAsSystem(() =>
@@ -776,25 +780,43 @@ function createAgentQueuedTurnScheduler({
return receipt.deliveryKey;
};
- const recover = (): Promise<number> => {
+ const schedule = async (turn: AgentQueuedTurnRecord): Promise<string> => {
+ try {
+ return await publish(turn);
+ } catch (error) {
+ loop?.wake();
+ throw error;
+ }
+ };
+
+ const recoverPass = (): Promise<{ repaired: number; idle: boolean }> => {
if (recovery != null) {
return recovery;
}
const task = (async () => {
const reconciliationNow = new Date();
const reconciliationClaimId = randomUUID();
- const [turns, quarantined] = await Promise.all([
- runAsSystem(() => methods.findQueuedTurnsNeedingDelivery(recoveryLimit)),
+ const activity = { found: false };
+ const discoveries = await Promise.allSettled([
+ runAsSystem(() => methods.findQueuedTurnsNeedingDelivery(recoveryLimit, activity)),
runAsSystem(() =>
- methods.claimQueuedTurnsForAdmissionReconciliation({
- claimId: reconciliationClaimId,
- claimBy: PROCESS_CLAIM_OWNER,
- now: reconciliationNow,
- leaseUntil: new Date(reconciliationNow.getTime() + RECONCILIATION_LEASE_MS),
- limit: recoveryLimit,
- }),
+ methods.claimQueuedTurnsForAdmissionReconciliation(
+ {
+ claimId: reconciliationClaimId,
+ claimBy: PROCESS_CLAIM_OWNER,
+ now: reconciliationNow,
+ leaseUntil: new Date(reconciliationNow.getTime() + RECONCILIATION_LEASE_MS),
+ limit: recoveryLimit,
+ },
+ activity,
+ ),
),
]);
+ // A rejected discovery must not release the single-flight guard while
+ // its sibling is still writing reconciliation leases in Mongo.
+ const [deliveries, reconciliations] = discoveries;
+ const turns = deliveries.status === 'fulfilled' ? deliveries.value : [];
+ const quarantined = reconciliations.status === 'fulfilled' ? reconciliations.value : [];
let repaired = 0;
for (const turn of quarantined) {
const deliveryKey = turn.deliveryKey;
@@ -805,8 +827,11 @@ function createAgentQueuedTurnScheduler({
) {
continue;
}
- const defer = () =>
- runAsSystem(() =>
+ const defer = async () => {
+ const availableAt = new Date(
+ Date.now() + reconciliationBackoff(turn.reconciliationAttempts),
+ );
+ const deferred = await runAsSystem(() =>
methods.deferAgentQueuedTurnAdmissionReconciliation({
user: turn.user,
...(turn.tenantId != null && { tenantId: turn.tenantId }),
@@ -815,11 +840,13 @@ function createAgentQueuedTurnScheduler({
deliveryKey,
claimId: reconciliationClaimId,
claimBy: PROCESS_CLAIM_OWNER,
- availableAt: new Date(
- Date.now() + reconciliationBackoff(turn.reconciliationAttempts),
- ),
+ availableAt,
}),
);
+ if (deferred) {
+ loop?.noteEligibleAt(availableAt);
+ }
+ };
try {
const isIndeterminate =
turn.terminalReceipt?.outcome === 'dead' &&
@@ -894,7 +921,7 @@ function createAgentQueuedTurnScheduler({
}
for (const turn of turns) {
try {
- await schedule(turn);
+ await publish(turn);
repaired += 1;
} catch (error) {
logger.warn(
@@ -904,7 +931,11 @@ function createAgentQueuedTurnScheduler({
);
}
}
- return repaired;
+ // Do not strand leases already acquired by the successful discovery just
+ // because its independent sibling failed. Process them, then report failure.
+ if (deliveries.status === 'rejected') throw deliveries.reason;
+ if (reconciliations.status === 'rejected') throw reconciliations.reason;
+ return { repaired, idle: !activity.found && turns.length === 0 && quarantined.length === 0 };
})();
recovery = task;
void task.then(
@@ -921,28 +952,48 @@ function createAgentQueuedTurnScheduler({
);
return task;
};
+ const recover = async (): Promise<number> => (await recoverPass()).repaired;
return {
schedule,
recover,
- initialize: async () => {
- await runAsSystem(() => methods.ensureAgentQueuedTurnIndexes());
- timer = setInterval(() => {
- void recover().catch((error: unknown) => {
- logger.warn('[agentQueuedTurns] Delivery recovery pass failed', error);
+ initialize: (options = {}) => {
+ if (initialization != null) {
+ return initialization;
+ }
+ if (stopped) {
+ return Promise.resolve();
+ }
+ const maxIdleIntervalMs =
+ options.maxIdleIntervalMs ??
+ Math.max(recoveryIntervalMs, DEFAULT_RECOVERY_MAX_IDLE_INTERVAL_MS);
+ const next = (async () => {
+ loop = createIdleRecoveryLoop({
+ intervalMs: recoveryIntervalMs,
+ maxIdleIntervalMs,
+ scan: async () => (await recoverPass()).idle,
+ onError: (error) =>
+ logger.warn('[agentQueuedTurns] Delivery recovery pass failed', error),
});
- }, recoveryIntervalMs);
- timer.unref?.();
- await recover().catch((error: unknown) => {
- logger.warn('[agentQueuedTurns] Initial delivery recovery pass failed', error);
+ await runAsSystem(() => methods.ensureAgentQueuedTurnIndexes());
+ if (!stopped) {
+ await loop.start();
+ }
+ })();
+ initialization = next.catch((error: unknown) => {
+ if (!stopped) {
+ initialization = undefined;
+ loop = undefined;
+ }
+ throw error;
});
+ return initialization;
},
stop: async () => {
- if (timer != null) {
- clearInterval(timer);
- timer = undefined;
- }
- await recovery;
+ stopped = true;
+ await loop?.stop();
+ await initialization?.catch(() => undefined);
+ await recovery?.catch(() => undefined);
},
};
}
diff --git a/packages/api/src/agents/recovery.spec.ts b/packages/api/src/agents/recovery.spec.ts
new file mode 100644
index 0000000000..506d8c0625
--- /dev/null
+++ b/packages/api/src/agents/recovery.spec.ts
@@ -0,0 +1,233 @@
+import { createIdleRecoveryLoop } from './recovery';
+
+describe('durable agent idle recovery', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ jest.setSystemTime(new Date('2026-09-01T00:00:00Z'));
+ });
+
+ afterEach(() => jest.useRealTimers());
+
+ it('backs off only after empty passes and wakes without waiting for the idle ceiling', async () => {
+ const scan = jest.fn(async () => true);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ await loop.start();
+ expect(scan).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(59_999);
+ expect(scan).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(1);
+ expect(scan).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(scan).toHaveBeenCalledTimes(3);
+ loop.wake();
+ await jest.advanceTimersByTimeAsync(0);
+ expect(scan).toHaveBeenCalledTimes(4);
+ await jest.advanceTimersByTimeAsync(59_999);
+ expect(scan).toHaveBeenCalledTimes(4);
+ await loop.stop();
+ });
+
+ it('never lets an in-flight empty result discard a wake or start overlapping scans', async () => {
+ let release!: (idle: boolean) => void;
+ const scan = jest.fn(
+ () =>
+ new Promise<boolean>((resolve) => {
+ release = resolve;
+ }),
+ );
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ const first = loop.start();
+ await jest.advanceTimersByTimeAsync(0);
+ loop.wake();
+ loop.wake();
+ expect(scan).toHaveBeenCalledTimes(1);
+ release(true);
+ await first;
+ await jest.advanceTimersByTimeAsync(0);
+ expect(scan).toHaveBeenCalledTimes(2);
+ release(true);
+ await jest.advanceTimersByTimeAsync(0);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(scan).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(scan).toHaveBeenCalledTimes(3);
+ release(false);
+ await loop.stop();
+ });
+
+ it('scans at the base cadence after a failed or nonempty pass', async () => {
+ const onError = jest.fn();
+ const scan = jest
+ .fn()
+ .mockResolvedValueOnce(true)
+ .mockRejectedValueOnce(new Error('mongo unavailable'))
+ .mockResolvedValueOnce(false)
+ .mockResolvedValue(true);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError,
+ });
+ await loop.start();
+ await jest.advanceTimersByTimeAsync(60_000);
+ expect(onError).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(scan).toHaveBeenCalledTimes(3);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(scan).toHaveBeenCalledTimes(4);
+ await loop.stop();
+ });
+
+ it('scans at known eligibility times rather than sleeping past deferred work', async () => {
+ const scan = jest.fn(async () => true);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ await loop.start();
+ const now = Date.now();
+ loop.noteEligibleAt(new Date(now + 14_000));
+ loop.noteEligibleAt(new Date(now + 5_000));
+ await jest.advanceTimersByTimeAsync(5_000);
+ expect(scan).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(9_000);
+ expect(scan).toHaveBeenCalledTimes(3);
+ await loop.stop();
+ });
+
+ it('discovers work written by another replica without a process-local wake', async () => {
+ let hasWork = false;
+ const scan = jest.fn(async () => !hasWork);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ await loop.start();
+ await jest.advanceTimersByTimeAsync(60_000);
+ expect(scan).toHaveBeenCalledTimes(2);
+ hasWork = true;
+ await jest.advanceTimersByTimeAsync(119_999);
+ expect(scan).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(1);
+ expect(scan).toHaveBeenCalledTimes(3);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(scan).toHaveBeenCalledTimes(4);
+ await loop.stop();
+ });
+
+ it('never postpones the bounded fallback scan when a later deadline is learned', async () => {
+ const scan = jest.fn(async () => true);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ await loop.start();
+ await jest.advanceTimersByTimeAsync(59_000);
+ loop.noteEligibleAt(new Date(Date.now() + 120_000));
+ await jest.advanceTimersByTimeAsync(1_000);
+ expect(scan).toHaveBeenCalledTimes(2);
+ await loop.stop();
+ });
+
+ it.each([false, true])(
+ 'rescans for a deadline that expires in-flight (stop: %s)',
+ async (stop) => {
+ let release!: (idle: boolean) => void;
+ const scan = jest
+ .fn<Promise<boolean>, []>()
+ .mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ release = resolve;
+ }),
+ )
+ .mockResolvedValue(true);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ const starting = loop.start();
+ await jest.advanceTimersByTimeAsync(0);
+ loop.noteEligibleAt(new Date(Date.now() + 5_000));
+ loop.noteEligibleAt(new Date(Date.now() + 20_000));
+ await jest.advanceTimersByTimeAsync(6_000);
+ expect(scan).toHaveBeenCalledTimes(1);
+ const stopping = stop ? loop.stop() : undefined;
+ release(false);
+ await starting;
+ await stopping;
+ await jest.advanceTimersByTimeAsync(0);
+ expect(scan).toHaveBeenCalledTimes(stop ? 1 : 2);
+ await jest.advanceTimersByTimeAsync(14_000);
+ expect(scan).toHaveBeenCalledTimes(stop ? 1 : 3);
+ await loop.stop();
+ expect(jest.getTimerCount()).toBe(0);
+ },
+ );
+
+ it('does not rescan or reset its idle timer when start is called twice', async () => {
+ const scan = jest.fn(async () => true);
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ await loop.start();
+ await jest.advanceTimersByTimeAsync(59_000);
+ await loop.start();
+ expect(scan).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(1_000);
+ expect(scan).toHaveBeenCalledTimes(2);
+ await loop.stop();
+ });
+
+ it('waits for its active scan on shutdown and never rearms a timer', async () => {
+ let release!: (value: boolean) => void;
+ const scan = jest.fn(
+ () =>
+ new Promise<boolean>((resolve) => {
+ release = resolve;
+ }),
+ );
+ const loop = createIdleRecoveryLoop({
+ intervalMs: 30_000,
+ maxIdleIntervalMs: 120_000,
+ scan,
+ onError: jest.fn(),
+ });
+ const started = loop.start();
+ await jest.advanceTimersByTimeAsync(0);
+ let stopped = false;
+ const stopping = loop.stop().then(() => {
+ stopped = true;
+ });
+ loop.wake();
+ expect(stopped).toBe(false);
+ release(true);
+ await Promise.all([started, stopping]);
+ await jest.advanceTimersByTimeAsync(300_000);
+ expect(stopped).toBe(true);
+ expect(scan).toHaveBeenCalledTimes(1);
+ expect(jest.getTimerCount()).toBe(0);
+ });
+});
diff --git a/packages/api/src/agents/recovery.ts b/packages/api/src/agents/recovery.ts
new file mode 100644
index 0000000000..93d3ca1b5d
--- /dev/null
+++ b/packages/api/src/agents/recovery.ts
@@ -0,0 +1,164 @@
+export interface IdleRecoveryLoopOptions {
+ intervalMs: number;
+ maxIdleIntervalMs: number;
+ /** True only after a completed scan confirmed no eligible work and no failures. */
+ scan: () => Promise<boolean>;
+ onError: (error: unknown) => void;
+}
+
+/** Mongo remains the authority: wakes are only hints, and every replica still scans
+ * within the idle ceiling so a crashed producer or a missed local wake is recovered. */
+export function createIdleRecoveryLoop({
+ intervalMs,
+ maxIdleIntervalMs,
+ scan,
+ onError,
+}: IdleRecoveryLoopOptions) {
+ if (!Number.isSafeInteger(intervalMs) || intervalMs <= 0) {
+ throw new TypeError('Recovery interval must be a positive integer');
+ }
+ if (!Number.isSafeInteger(maxIdleIntervalMs) || maxIdleIntervalMs < intervalMs) {
+ throw new TypeError('Maximum idle recovery interval must be at least the recovery interval');
+ }
+
+ let started = false;
+ let stopped = false;
+ let idleStreak = 0;
+ let nextPollAt = 0;
+ let wakePending = false;
+ let generation = 0;
+ let timer: NodeJS.Timeout | undefined;
+ let active: Promise<void> | undefined;
+ const deadlines: number[] = [];
+ const MAX_DEADLINES = 64;
+
+ const schedule = (): void => {
+ if (!started || stopped || active != null) {
+ return;
+ }
+ if (timer != null) {
+ clearTimeout(timer);
+ }
+ const now = Date.now();
+ let delay = Math.max(0, nextPollAt - now);
+ if (deadlines.length > 0) {
+ delay = Math.min(delay, Math.max(0, deadlines[0] - now));
+ }
+ timer = setTimeout(() => {
+ timer = undefined;
+ void runNow();
+ }, delay);
+ timer.unref?.();
+ };
+
+ const runNow = (): Promise<void> => {
+ if (stopped || !started) {
+ return Promise.resolve();
+ }
+ if (active != null) {
+ return active;
+ }
+ if (timer != null) {
+ clearTimeout(timer);
+ timer = undefined;
+ }
+ // This scan can cover deadlines already due when it starts. Deadlines
+ // learned or becoming due while it runs must survive for a follow-up scan.
+ const now = Date.now();
+ while (deadlines.length > 0 && deadlines[0] <= now) {
+ deadlines.shift();
+ }
+ const before = generation;
+ const current = Promise.resolve()
+ .then(scan)
+ .then(
+ (idle) => {
+ idleStreak = idle && before === generation ? idleStreak + 1 : 0;
+ },
+ (error: unknown) => {
+ idleStreak = 0;
+ onError(error);
+ },
+ )
+ .finally(() => {
+ if (active === current) {
+ active = undefined;
+ }
+ if (stopped) {
+ return;
+ }
+ nextPollAt =
+ Date.now() + Math.min(intervalMs * 2 ** Math.min(idleStreak, 30), maxIdleIntervalMs);
+ if (wakePending) {
+ wakePending = false;
+ queueMicrotask(() => void runNow());
+ } else {
+ schedule();
+ }
+ });
+ active = current;
+ return current;
+ };
+
+ return {
+ start: (): Promise<void> => {
+ if (stopped) {
+ return Promise.resolve();
+ }
+ if (started) {
+ return active ?? Promise.resolve();
+ }
+ started = true;
+ return runNow();
+ },
+ /** Coalesce notifications while a scan is in progress; none can be erased
+ * by that scan's earlier empty result. */
+ wake: (): void => {
+ if (stopped) {
+ return;
+ }
+ generation += 1;
+ idleStreak = 0;
+ if (active != null) {
+ wakePending = true;
+ } else if (started) {
+ void runNow();
+ }
+ },
+ noteEligibleAt: (at: Date): void => {
+ const time = at.getTime();
+ if (!Number.isFinite(time) || stopped) {
+ return;
+ }
+ if (time <= Date.now()) {
+ if (started) {
+ generation += 1;
+ idleStreak = 0;
+ if (active != null) {
+ wakePending = true;
+ } else {
+ void runNow();
+ }
+ }
+ return;
+ }
+ const index = deadlines.findIndex((value) => value >= time);
+ if (index !== -1 && deadlines[index] === time) {
+ return;
+ }
+ deadlines.splice(index < 0 ? deadlines.length : index, 0, time);
+ if (deadlines.length > MAX_DEADLINES) {
+ deadlines.pop();
+ }
+ schedule();
+ },
+ stop: async (): Promise<void> => {
+ stopped = true;
+ if (timer != null) {
+ clearTimeout(timer);
+ timer = undefined;
+ }
+ await active;
+ },
+ };
+}
diff --git a/packages/api/src/agents/triggers/README.md b/packages/api/src/agents/triggers/README.md
index 93abb3f4bb..faef699ef5 100644
--- a/packages/api/src/agents/triggers/README.md
+++ b/packages/api/src/agents/triggers/README.md
@@ -47,6 +47,7 @@ await enqueueAgentTrigger(
## Guarantees
- Mongo owns queue state, leases, retry history, and dead letters across restarts and replicas.
+- Each replica retains bounded Mongo polling even when idle. See **Idle recovery** below for wake-ups, deadlines, and configuration.
- A fresh token fences every claim, including reclaims by the same process.
- A delivery is at-least-once. Fire, continue, and steer admission reuse the envelope's stable idempotency
identity, so ambiguous retries do not duplicate accepted work.
@@ -71,6 +72,37 @@ await enqueueAgentTrigger(
`getAgentTriggerDeadLetters` and `requeueAgentTrigger` are intentionally trusted in-process
operations. Exposing them through an admin API requires a separate authorization and audit layer.
+## Idle recovery
+
+Queued-turn and maintenance scans start at a 30-second cadence and double the next idle wait
+up to two minutes after confirmed-empty discovery. Inspected reservations, reconciliation
+candidates, outstanding cleanup markers, full legacy-receipt pages, and failed scans do not
+count as empty. A failed discovery waits for its sibling Mongo operation to settle before a
+new pass or shutdown can complete; successfully acquired work still gets processed.
+
+Local failed publication and requeue, unfinished terminal finalization, and purge markers wake
+maintenance without waiting for an idle timer. A committed root success or dead letter remains
+authoritative when its inline cleanup fails. Healthy terminal writes do not wake a full sweep.
+Known reconciliation deadlines cap the next wait, including deadlines that expire while a scan
+is in flight. Notifications coalesce behind one active scan; shutdown prevents a follow-up.
+
+Every replica still scans without notifications, so a crashed producer or a missed cross-replica
+wake cannot strand durable work. The cap limits **idle sleep**, not end-to-end recovery time:
+scan duration, pagination, active leases, and persistence failures can add time. Delivery claims
+retain their separate 15-second idle cap and immediate local enqueue/requeue wake-ups.
+
+Configure `endpoints.agents.eventDriven.idlePolling` in `librechat.yaml`, for either standard or
+experimental clustered startup:
+
+| Setting | Default (ms) | Allowed range (ms) |
+| --- | ---: | ---: |
+| `queuedTurnMaxIntervalMs` | 120000 | 30000–300000 |
+| `maintenanceMaxIntervalMs` | 120000 | 30000–300000 |
+| `deliveryMaxIntervalMs` | 15000 | 1000–300000 |
+
+Setting a recovery cap to `30000` restores its original fixed recovery frequency. No stored-data
+migration is needed; optional activity reporting preserves the existing numeric/boolean results.
+
## Remote event ingress
Authenticated controllers and source adapters can enqueue the same durable envelope through
diff --git a/packages/api/src/agents/triggers/delivery.integration.spec.ts b/packages/api/src/agents/triggers/delivery.integration.spec.ts
index bc78de4fa1..364edf0be9 100644
--- a/packages/api/src/agents/triggers/delivery.integration.spec.ts
+++ b/packages/api/src/agents/triggers/delivery.integration.spec.ts
@@ -4,6 +4,7 @@ import { createMethods, createModels } from '@librechat/data-schemas';
import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './service';
import type { AgentTriggerFetch } from './host';
import { __resetShutdownStateForTests } from '../../app/shutdown';
+import { prepareAgentTriggerDelivery } from './delivery';
import { createAgentTriggerEnvelope } from './envelope';
import { createAgentTriggerService } from './service';
@@ -83,6 +84,79 @@ async function eventuallySucceeded(deliveryKey: string) {
}
describe('durable trigger delivery integration', () => {
+ it('recovers an abandoned publication across two idle workers without a producer wake', async () => {
+ const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
+ async () =>
+ new Response(
+ JSON.stringify({
+ status: 'started',
+ streamId: 'cross-replica',
+ conversationId: 'cross-replica',
+ generationCreatedAt: 25,
+ }),
+ { status: 200, headers: { 'content-type': 'application/json' } },
+ ),
+ );
+ const bundles = [createMethods(mongoose), createMethods(mongoose)];
+ const scans = bundles.map((methods) =>
+ jest.spyOn(methods, 'recoverAgentTriggerLanePublications'),
+ );
+ const workers = bundles.map((methods) =>
+ createAgentTriggerService({
+ methods: methods as typeof methods & AgentTriggerDeliveryPersistence,
+ fetch: fetcher,
+ mintToken: () => 'trigger-token',
+ purgeRecoveryIntervalMs: 20,
+ deliveryOptions: {
+ concurrency: 1,
+ tickMs: 5,
+ maxIdleTickMs: 40,
+ },
+ }),
+ );
+ const until = async (condition: () => boolean | Promise<boolean>) => {
+ const deadline = Date.now() + 5_000;
+ while (!(await condition())) {
+ if (Date.now() >= deadline) throw new Error('Timed out waiting for cross-replica recovery');
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ };
+ try {
+ for (const worker of workers)
+ await worker.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ idlePolling: { maintenanceMaxIntervalMs: 80 },
+ });
+ await until(() => scans.every((scan) => scan.mock.calls.length >= 3));
+ const prepared = prepareAgentTriggerDelivery(envelope(), { orderingKey: 'crashed-producer' });
+ // A producer died after its staging write. Neither worker receives enqueue()
+ // or wake(); both must discover and fence publication via their Mongo fallback.
+ await mongoose.models.AgentTriggerDelivery.create({
+ ...prepared,
+ laneSequence: 0,
+ status: 'staging',
+ attempts: 0,
+ requeueCount: 0,
+ claimAvailableAt: prepared.availableAt,
+ stagingRecoveryAt: new Date(),
+ });
+ await until(
+ async () =>
+ (await bundles[0].getAgentTriggerDelivery(prepared.deliveryKey))?.status === 'succeeded',
+ );
+ expect(fetcher).toHaveBeenCalledTimes(1);
+ const delivered = await bundles[1].getAgentTriggerDelivery(prepared.deliveryKey);
+ expect(delivered).toMatchObject({
+ status: 'succeeded',
+ attempts: 1,
+ history: [{ outcome: 'succeeded' }],
+ });
+ } finally {
+ await Promise.all(workers.map((worker) => worker.stop()));
+ scans.forEach((scan) => scan.mockRestore());
+ }
+ });
+
it('moves a trusted envelope through Mongo, the lease worker, and host admission', async () => {
const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
async () =>
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index f3b6b4c103..8d571b0f92 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -7,9 +7,11 @@ import {
SYSTEM_TENANT_ID,
} from '@librechat/data-schemas';
import type { AgentTriggerDeliveryPersistence, AgentTriggerStoredRecord } from './service';
+import type { AgentTriggerDeliveryStore } from './engine';
import { AgentTriggerServiceUnavailableError, createAgentTriggerService } from './service';
import { __resetShutdownStateForTests } from '../../app/shutdown';
import { createAgentTriggerEnvelope } from './envelope';
+import * as deliveryEngineModule from './engine';
jest.mock('@librechat/data-schemas', () => {
const actual = jest.requireActual('@librechat/data-schemas');
@@ -321,6 +323,183 @@ describe('durable agent trigger service', () => {
expect(deleteAgentTriggerDeliveriesByUser).toHaveBeenCalledWith('507f1f77bcf86cd799439011');
});
+ it.each(['complete', 'dead', 'retire'] as const)(
+ 'wakes for unfinished %s finalization, but not healthy terminal writes',
+ async (operation) => {
+ jest.useFakeTimers();
+ let service: ReturnType<typeof createAgentTriggerService> | undefined;
+ try {
+ let store!: AgentTriggerDeliveryStore;
+ const create = deliveryEngineModule.createAgentTriggerDeliveryEngine;
+ jest
+ .spyOn(deliveryEngineModule, 'createAgentTriggerDeliveryEngine')
+ .mockImplementation((deps, options) => {
+ store = deps.store;
+ return create(deps, options);
+ });
+ let unfinished = false;
+ const terminal = jest.fn(async (_input: unknown, recovery?: { required: boolean }) => {
+ if (recovery != null) recovery.required = unfinished;
+ return true;
+ });
+ const methods = deliveryMethods({
+ completeAgentTriggerDelivery: terminal,
+ deadLetterAgentTriggerDelivery: terminal,
+ retireAgentTriggerDelivery: terminal,
+ });
+ service = createAgentTriggerService({ methods, deliveryOptions: { tickMs: 300_000 } });
+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
+ await jest.advanceTimersByTimeAsync(180_000);
+ const count = (methods.recoverAgentTriggerBatchReceipts as jest.Mock).mock.calls.length;
+ const input = {
+ id: 'row',
+ workerId: 'worker',
+ claimToken: 'token',
+ attempt: 1,
+ settledAt: new Date(),
+ };
+ const finish = () => {
+ if (operation === 'complete') {
+ return store.complete({
+ ...input,
+ result: { mode: 'fire', status: 'started', conversationId: 'conversation-1' },
+ });
+ }
+ if (operation === 'dead') {
+ return store.dead({
+ ...input,
+ error: {
+ code: 'FAILED',
+ message: 'failed',
+ retryable: false,
+ certainty: 'definite',
+ attemptedAt: new Date(),
+ },
+ });
+ }
+ return service!.retire('delivery-key', 'source', 'cancelled');
+ };
+ await expect(finish()).resolves.toBe(true);
+ await jest.advanceTimersByTimeAsync(0);
+ expect(methods.recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(count);
+ unfinished = true;
+ await expect(finish()).resolves.toBe(true);
+ await jest.advanceTimersByTimeAsync(0);
+ expect(methods.recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(count + 1);
+ expect(getTenantId()).not.toBe(SYSTEM_TENANT_ID);
+ } finally {
+ await service?.stop();
+ jest.restoreAllMocks();
+ jest.useRealTimers();
+ }
+ },
+ );
+
+ it.each(['enqueue', 'requeue'] as const)(
+ 'wakes maintenance after an ambiguous %s write failure',
+ async (operation) => {
+ jest.useFakeTimers();
+ let service: ReturnType<typeof createAgentTriggerService> | undefined;
+ try {
+ const methods = deliveryMethods();
+ (methods.enqueueAgentTriggerDelivery as jest.Mock).mockRejectedValue(
+ new Error('publication interrupted'),
+ );
+ (methods.requeueAgentTriggerDelivery as jest.Mock).mockRejectedValue(
+ new Error('publication interrupted'),
+ );
+ service = createAgentTriggerService({ methods, deliveryOptions: { tickMs: 300_000 } });
+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
+ await jest.advanceTimersByTimeAsync(180_000);
+ const count = (methods.recoverAgentTriggerLanePublications as jest.Mock).mock.calls.length;
+ await expect(
+ operation === 'enqueue' ? service.enqueue(envelope()) : service.requeue('row'),
+ ).rejects.toThrow('publication interrupted');
+ await jest.advanceTimersByTimeAsync(0);
+ expect(methods.recoverAgentTriggerLanePublications).toHaveBeenCalledTimes(count + 1);
+ } finally {
+ await service?.stop();
+ jest.useRealTimers();
+ }
+ },
+ );
+
+ it('backs off empty maintenance scans and wakes for new cleanup work', async () => {
+ jest.useFakeTimers();
+ try {
+ const recoverAgentTriggerUserPurges = jest.fn(async () => 0);
+ const methods = deliveryMethods({ recoverAgentTriggerUserPurges });
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 300_000 },
+ });
+ await service.initialize({
+ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
+ });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(60_000);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(3);
+ await service.prepareUserPurge('507f1f77bcf86cd799439011', new Date());
+ await jest.advanceTimersByTimeAsync(0);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(4);
+ await service.stop();
+ await jest.advanceTimersByTimeAsync(120_000);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(4);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('keeps the base cadence when a recovery marker was inspected but no repair completed', async () => {
+ jest.useFakeTimers();
+ try {
+ const recoverAgentTriggerBatchReceipts = jest.fn(
+ async (_limit?: number, activity?: { found: boolean }) => {
+ if (activity != null) activity.found = true;
+ return 0;
+ },
+ );
+ const service = createAgentTriggerService({
+ methods: deliveryMethods({ recoverAgentTriggerBatchReceipts }),
+ deliveryOptions: { concurrency: 1, tickMs: 300_000 },
+ });
+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(recoverAgentTriggerBatchReceipts).toHaveBeenCalledTimes(2);
+ await service.stop();
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('does not mistake an isolated maintenance failure for an empty scan', async () => {
+ jest.useFakeTimers();
+ try {
+ const recoverAgentTriggerUserPurges = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('mongo unavailable'))
+ .mockResolvedValue(0);
+ const methods = deliveryMethods({ recoverAgentTriggerUserPurges });
+ const service = createAgentTriggerService({
+ methods,
+ deliveryOptions: { concurrency: 1, tickMs: 300_000 },
+ });
+ await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(30_000);
+ expect(recoverAgentTriggerUserPurges).toHaveBeenCalledTimes(2);
+ await service.stop();
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
it('arms and disarms post-commit purge recovery in system context', async () => {
const prepareAgentTriggerUserPurge = jest.fn(async () => {
expect(getTenantId()).toBe(SYSTEM_TENANT_ID);
@@ -385,7 +564,11 @@ describe('durable agent trigger service', () => {
address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
});
- expect(expireLegacyAgentEventActorReceipts).toHaveBeenCalledWith(expect.any(Date), 17);
+ expect(expireLegacyAgentEventActorReceipts).toHaveBeenCalledWith(
+ expect.any(Date),
+ 17,
+ expect.objectContaining({ found: false }),
+ );
await service.stop();
});
@@ -405,7 +588,10 @@ describe('durable agent trigger service', () => {
deliveryOptions: { concurrency: 1, tickMs: 60_000 },
});
await service.initialize({ address: { address: '127.0.0.1', family: 'IPv4', port: 3080 } });
- expect(reclaimCheckpointDeletions).toHaveBeenCalledWith(17);
+ expect(reclaimCheckpointDeletions).toHaveBeenCalledWith(
+ 17,
+ expect.objectContaining({ found: false }),
+ );
let stopped = false;
const stop = service.stop().then(() => {
stopped = true;
@@ -545,6 +731,7 @@ describe('durable agent trigger service', () => {
reason: 'result unavailable',
onlyIfUnclaimed: true,
}),
+ { required: false },
);
await expect(
service.retire('trigger_1', 'background-tool-completion', 'dead recovery', {
@@ -553,6 +740,7 @@ describe('durable agent trigger service', () => {
).resolves.toBe(true);
expect(retireAgentTriggerDelivery).toHaveBeenLastCalledWith(
expect.objectContaining({ onlyIfDead: true }),
+ { required: false },
);
await service.stop();
});
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index 18f7b32c69..a37e1821f3 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -32,16 +32,23 @@ import { prepareAgentTriggerDelivery } from './delivery';
import { selfOriginFromAddress } from '../../app/origin';
import { createAgentTriggerExecutionHost } from './host';
import { parseAgentTriggerEnvelope } from './envelope';
+import { createIdleRecoveryLoop } from '../recovery';
export const AGENT_TRIGGER_TOKEN_TTL = '60s';
const DEFAULT_USER_DRAIN_TIMEOUT_MS = 35_000;
const DEFAULT_USER_DRAIN_POLL_MS = 100;
const DEFAULT_PURGE_RECOVERY_INTERVAL_MS = 30_000;
+const DEFAULT_PURGE_RECOVERY_MAX_IDLE_INTERVAL_MS = 2 * 60_000;
const DEFAULT_PURGE_RECOVERY_LIMIT = 25;
export interface AgentTriggerServiceOptions {
completionResultBatchSize?: number;
address?: BoundAddress | string | null;
+ idlePolling?: {
+ queuedTurnMaxIntervalMs?: number;
+ maintenanceMaxIntervalMs?: number;
+ deliveryMaxIntervalMs?: number;
+ };
}
export interface AgentTriggerServiceDeps {
@@ -57,7 +64,7 @@ export interface AgentTriggerServiceDeps {
userDrainPollMs?: number;
purgeRecoveryIntervalMs?: number;
purgeRecoveryLimit?: number;
- reclaimCheckpointDeletions?: (limit: number) => Promise<number>;
+ reclaimCheckpointDeletions?: (limit: number, activity?: { found: boolean }) => Promise<number>;
supportsDetachedActionCompletion?: () => boolean;
settleSourceBeforeDeadLetter?: AgentTriggerDeliveryEngineDeps['settleSourceBeforeDeadLetter'];
}
@@ -111,14 +118,14 @@ export interface AgentTriggerDeliveryPersistence {
releaseAgentTriggerDelivery: AgentTriggerDeliveryStore['release'];
beginAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['beginAttempt'];
deferAgentTriggerDeliveryAttempt: AgentTriggerDeliveryStore['defer'];
- completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];
+ completeAgentTriggerDelivery: AgentTriggerDeliveryMethods['completeAgentTriggerDelivery'];
retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];
renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];
persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
- deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];
+ deadLetterAgentTriggerDelivery: AgentTriggerDeliveryMethods['deadLetterAgentTriggerDelivery'];
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
getAgentTriggerDeliveryStatus: (
deliveryKey: string,
@@ -132,17 +139,30 @@ export interface AgentTriggerDeliveryPersistence {
availableAt: Date,
) => Promise<AgentTriggerStoredRecord | null>;
countActiveAgentTriggerDeliveriesByUser: (userId: string, now: Date) => Promise<number>;
- recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;
- recoverAgentTriggerBatchReceipts: (limit?: number) => Promise<number>;
- reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;
+ recoverAgentTriggerLanePublications: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
+ recoverAgentTriggerBatchReceipts: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
+ reclaimInactiveAgentTriggerLanes: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
prepareAgentTriggerUserPurge: (
userId: string,
fenceStartedAt: Date,
tenantId?: string,
) => Promise<void>;
cancelAgentTriggerUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
- recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
- expireLegacyAgentEventActorReceipts?: (now: Date, limit?: number) => Promise<number>;
+ recoverAgentTriggerUserPurges: (limit?: number, activity?: { found: boolean }) => Promise<number>;
+ expireLegacyAgentEventActorReceipts?: (
+ now: Date,
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
deleteAgentTriggerDeliveriesByUser: (userId: string) => Promise<void>;
}
@@ -193,9 +213,29 @@ export interface AgentTriggerService {
purgeUser: (userId: string) => Promise<void>;
}
+/** A producer can commit durable state before an inline finalizer fails. Keep
+ * its authoritative result intact, but wake maintenance for the remaining marker.
+ * Rejections are uncertain commits, so they also request recovery. Healthy writes
+ * do not turn every delivered event into a full maintenance sweep. */
+async function withMaintenanceRecovery<T>(
+ operation: (recovery: { required: boolean }) => Promise<T>,
+ wake: () => void,
+): Promise<T> {
+ const recovery = { required: false };
+ try {
+ return await operation(recovery);
+ } catch (error) {
+ recovery.required = true;
+ throw error;
+ } finally {
+ if (recovery.required) wake();
+ }
+}
+
function createDeliveryStore(
methods: AgentTriggerDeliveryPersistence,
supportsDetachedActionCompletion: () => boolean,
+ wakeMaintenance: () => void,
): AgentTriggerDeliveryStore {
return {
claimNext: (input) =>
@@ -215,9 +255,17 @@ function createDeliveryStore(
release: methods.releaseAgentTriggerDelivery,
beginAttempt: methods.beginAgentTriggerDeliveryAttempt,
defer: methods.deferAgentTriggerDeliveryAttempt,
- complete: methods.completeAgentTriggerDelivery,
+ complete: (input) =>
+ withMaintenanceRecovery(
+ (recovery) => methods.completeAgentTriggerDelivery(input, recovery),
+ wakeMaintenance,
+ ),
retry: methods.retryAgentTriggerDelivery,
- dead: methods.deadLetterAgentTriggerDelivery,
+ dead: (input) =>
+ withMaintenanceRecovery(
+ (recovery) => methods.deadLetterAgentTriggerDelivery(input, recovery),
+ wakeMaintenance,
+ ),
};
}
@@ -277,8 +325,8 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
let backgroundCompletionResultBatchSize = 8;
let deliveryEngine: AgentTriggerDeliveryEngine | undefined;
let initializePromise: Promise<void> | undefined;
- let purgeRecoveryPromise: Promise<void> | undefined;
- let purgeRecoveryTimer: NodeJS.Timeout | undefined;
+ let purgeRecoveryPromise: Promise<boolean> | undefined;
+ let purgeRecoveryLoop: ReturnType<typeof createIdleRecoveryLoop> | undefined;
let deliveryReady = false;
let stopping = false;
const isPrincipalActive = deps.isPrincipalActive;
@@ -362,9 +410,9 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
};
- const recoverPurges = (): Promise<void> => {
+ const recoverPurges = (): Promise<boolean> => {
if (deps.methods == null || stopping) {
- return Promise.resolve();
+ return Promise.resolve(false);
}
if (purgeRecoveryPromise != null) {
return purgeRecoveryPromise;
@@ -378,14 +426,20 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
* half-recovered batch clears a request that a later successful recovery
* can no longer re-arm, retaining the lane permanently — so reclamation
* still waits for a batch-recovery pass that did not fail. */
- const isolated = (label: string, run: () => Promise<number>): Promise<number> =>
- run().catch((error) => {
+ let failed = false;
+ const activity = { found: false };
+ const isolated = async (label: string, run: () => Promise<number>): Promise<number> => {
+ try {
+ return await run();
+ } catch (error) {
+ failed = true;
logger.error(
`[agent-triggers] durable delivery maintenance step failed (${label}):`,
error,
);
return 0;
- });
+ }
+ };
const current = runAsSystem(async () => {
const [
purgedUsers,
@@ -394,13 +448,16 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
expiredLegacyActorReceipts,
retiredCheckpointDeletions,
] = await Promise.all([
- isolated('user purges', () => methods.recoverAgentTriggerUserPurges(purgeRecoveryLimit)),
+ isolated('user purges', () =>
+ methods.recoverAgentTriggerUserPurges(purgeRecoveryLimit, activity),
+ ),
isolated('lane publications', () =>
- methods.recoverAgentTriggerLanePublications(purgeRecoveryLimit),
+ methods.recoverAgentTriggerLanePublications(purgeRecoveryLimit, activity),
),
- methods.recoverAgentTriggerBatchReceipts(purgeRecoveryLimit).then(
+ methods.recoverAgentTriggerBatchReceipts(purgeRecoveryLimit, activity).then(
(count) => ({ succeeded: true as const, count }),
(error) => {
+ failed = true;
logger.error(
'[agent-triggers] durable delivery maintenance step failed (batch receipts):',
error,
@@ -411,18 +468,22 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
isolated(
'legacy actor receipts',
() =>
- methods.expireLegacyAgentEventActorReceipts?.(new Date(), purgeRecoveryLimit) ??
- Promise.resolve(0),
+ methods.expireLegacyAgentEventActorReceipts?.(
+ new Date(),
+ purgeRecoveryLimit,
+ activity,
+ ) ?? Promise.resolve(0),
),
isolated(
'checkpoint deletion evidence',
- () => deps.reclaimCheckpointDeletions?.(purgeRecoveryLimit) ?? Promise.resolve(0),
+ () =>
+ deps.reclaimCheckpointDeletions?.(purgeRecoveryLimit, activity) ?? Promise.resolve(0),
),
]);
const recoveredBatches = batchRecovery.count;
const reclaimedLanes = batchRecovery.succeeded
? await isolated('lane reclamation', () =>
- methods.reclaimInactiveAgentTriggerLanes(purgeRecoveryLimit),
+ methods.reclaimInactiveAgentTriggerLanes(purgeRecoveryLimit, activity),
)
: 0;
if (publishedLanes > 0) {
@@ -445,9 +506,20 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
retiredCheckpointDeletions,
});
}
+ return (
+ !failed &&
+ !activity.found &&
+ purgedUsers === 0 &&
+ publishedLanes === 0 &&
+ recoveredBatches === 0 &&
+ reclaimedLanes === 0 &&
+ expiredLegacyActorReceipts === 0 &&
+ retiredCheckpointDeletions === 0
+ );
})
.catch((error) => {
logger.error('[agent-triggers] durable delivery maintenance failed:', error);
+ return false;
})
.finally(() => {
if (purgeRecoveryPromise === current) {
@@ -459,21 +531,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
};
const startPurgeRecovery = (): void => {
- if (purgeRecoveryTimer != null) {
- return;
- }
- void recoverPurges();
- purgeRecoveryTimer = setInterval(() => void recoverPurges(), purgeRecoveryIntervalMs);
- purgeRecoveryTimer.unref();
+ void purgeRecoveryLoop?.start();
};
const stop = async (): Promise<void> => {
stopping = true;
deliveryReady = false;
- if (purgeRecoveryTimer != null) {
- clearInterval(purgeRecoveryTimer);
- purgeRecoveryTimer = undefined;
- }
+ await purgeRecoveryLoop?.stop();
await initializePromise?.catch(() => undefined);
await deliveryEngine?.stop();
await purgeRecoveryPromise?.catch(() => undefined);
@@ -506,6 +570,15 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
const methods = deps.methods;
initializePromise = runAsSystem(async () => {
requireDeliveryOrigin(boundOrigin);
+ purgeRecoveryLoop = createIdleRecoveryLoop({
+ intervalMs: purgeRecoveryIntervalMs,
+ maxIdleIntervalMs:
+ options.idlePolling?.maintenanceMaxIntervalMs ??
+ Math.max(purgeRecoveryIntervalMs, DEFAULT_PURGE_RECOVERY_MAX_IDLE_INTERVAL_MS),
+ scan: recoverPurges,
+ onError: (error) =>
+ logger.error('[agent-triggers] durable delivery maintenance failed:', error),
+ });
await methods.ensureAgentTriggerDeliveryIndexes();
if (stopping || isShutdownInProgress()) {
throw new AgentTriggerServiceUnavailableError(
@@ -514,13 +587,20 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
deliveryEngine = createAgentTriggerDeliveryEngine(
{
- store: createDeliveryStore(methods, supportsDetachedActionCompletion),
+ store: createDeliveryStore(methods, supportsDetachedActionCompletion, () =>
+ purgeRecoveryLoop?.wake(),
+ ),
dispatch: dispatchForActivePrincipal,
...(deps.settleSourceBeforeDeadLetter != null && {
settleSourceBeforeDeadLetter: deps.settleSourceBeforeDeadLetter,
}),
},
- deps.deliveryOptions,
+ {
+ ...deps.deliveryOptions,
+ ...(options.idlePolling?.deliveryMaxIntervalMs != null && {
+ maxIdleTickMs: options.idlePolling.deliveryMaxIntervalMs,
+ }),
+ },
);
deliveryReady = true;
deliveryEngine.start();
@@ -546,7 +626,10 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
};
await requireActivePrincipal(String(prepared.user));
const queued = await runAsSystem(async () =>
- methods.enqueueAgentTriggerDelivery(durableDelivery),
+ withMaintenanceRecovery(
+ () => methods.enqueueAgentTriggerDelivery(durableDelivery),
+ () => purgeRecoveryLoop?.wake(),
+ ),
);
try {
await requireActivePrincipal(String(prepared.user));
@@ -589,7 +672,11 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
runAsSystem(async () => requireMethods().getAgentTriggerDeadLetters(limit)),
requeue: (id, availableAt = new Date()) =>
runAsSystem(async () => {
- const revived = await requireMethods().requeueAgentTriggerDelivery(id, availableAt);
+ const methods = requireMethods();
+ const revived = await withMaintenanceRecovery(
+ () => methods.requeueAgentTriggerDelivery(id, availableAt),
+ () => purgeRecoveryLoop?.wake(),
+ );
if (revived != null) {
if (availableAt.getTime() > Date.now()) {
deliveryEngine?.noteEligibleAt(availableAt);
@@ -601,14 +688,22 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}),
retire: (deliveryKey, sourceId, reason, options) =>
runAsSystem(async () => {
- const retired = await requireCleanupMethods().retireAgentTriggerDelivery({
- deliveryKey,
- sourceId,
- reason,
- settledAt: new Date(),
- ...(options?.onlyIfUnclaimed === true ? { onlyIfUnclaimed: true } : {}),
- ...(options?.onlyIfDead === true ? { onlyIfDead: true } : {}),
- });
+ const methods = requireCleanupMethods();
+ const retired = await withMaintenanceRecovery(
+ (recovery) =>
+ methods.retireAgentTriggerDelivery(
+ {
+ deliveryKey,
+ sourceId,
+ reason,
+ settledAt: new Date(),
+ ...(options?.onlyIfUnclaimed === true ? { onlyIfUnclaimed: true } : {}),
+ ...(options?.onlyIfDead === true ? { onlyIfDead: true } : {}),
+ },
+ recovery,
+ ),
+ () => purgeRecoveryLoop?.wake(),
+ );
if (retired) {
deliveryEngine?.wake();
}
@@ -640,9 +735,14 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}),
drainUser,
prepareUserPurge: (userId, fenceStartedAt, tenantId) =>
- runAsSystem(async () =>
- requireCleanupMethods().prepareAgentTriggerUserPurge(userId, fenceStartedAt, tenantId),
- ),
+ runAsSystem(async () => {
+ const methods = requireCleanupMethods();
+ await withMaintenanceRecovery(
+ () => methods.prepareAgentTriggerUserPurge(userId, fenceStartedAt, tenantId),
+ () => purgeRecoveryLoop?.wake(),
+ );
+ purgeRecoveryLoop?.wake();
+ }),
cancelUserPurge: (userId, fenceStartedAt) =>
runAsSystem(async () =>
requireCleanupMethods().cancelAgentTriggerUserPurge(userId, fenceStartedAt),
@@ -651,6 +751,13 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
// shutdown has begun. Persistence remains usable even though admissions
// and the delivery engine are deliberately no longer ready.
purgeUser: (userId) =>
- runAsSystem(async () => requireCleanupMethods().deleteAgentTriggerDeliveriesByUser(userId)),
+ runAsSystem(async () => {
+ const methods = requireCleanupMethods();
+ await withMaintenanceRecovery(
+ () => methods.deleteAgentTriggerDeliveriesByUser(userId),
+ () => purgeRecoveryLoop?.wake(),
+ );
+ purgeRecoveryLoop?.wake();
+ }),
};
}
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index d41e03d260..02676ab8ad 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -673,6 +673,31 @@ describe('agent background completion batch config', () => {
});
describe('agent event runtime config', () => {
+ it('defaults and bounds durable-worker idle polling intervals', () => {
+ const parsed = configSchema.parse({
+ version: '1.0',
+ endpoints: { agents: { eventDriven: { idlePolling: {} } } },
+ });
+ expect(parsed.endpoints?.agents?.eventDriven?.idlePolling).toEqual({
+ deliveryMaxIntervalMs: 15_000,
+ queuedTurnMaxIntervalMs: 120_000,
+ maintenanceMaxIntervalMs: 120_000,
+ });
+ for (const [key, value] of [
+ ['deliveryMaxIntervalMs', 0],
+ ['queuedTurnMaxIntervalMs', 29_999],
+ ['maintenanceMaxIntervalMs', 300_001],
+ ['maintenanceMaxIntervalMs', 30_000.5],
+ ] as const) {
+ expect(
+ configSchema.safeParse({
+ version: '1.0',
+ endpoints: { agents: { eventDriven: { idlePolling: { [key]: value } } } },
+ }).success,
+ ).toBe(false);
+ }
+ });
+
it('accepts the routing choice and ignores removed rollout fields', () => {
const result = configSchema.safeParse({
version: '1.0',
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 691b48ac1e..698a5a6406 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -1512,6 +1512,33 @@ export const agentsEndpointSchema = baseEndpointSchema
eventDriven: z
.object({
selfUrl: z.string().url().optional(),
+ /** Every replica keeps polling Mongo as a crash-recovery fallback. Wakes
+ * keep local delivery prompt; these caps bound work missed across replicas. */
+ idlePolling: z
+ .object({
+ deliveryMaxIntervalMs: z
+ .number()
+ .int()
+ .min(1_000)
+ .max(300_000)
+ .optional()
+ .default(15_000),
+ queuedTurnMaxIntervalMs: z
+ .number()
+ .int()
+ .min(30_000)
+ .max(300_000)
+ .optional()
+ .default(120_000),
+ maintenanceMaxIntervalMs: z
+ .number()
+ .int()
+ .min(30_000)
+ .max(300_000)
+ .optional()
+ .default(120_000),
+ })
+ .optional(),
})
.optional(),
/** Conversational background-task delivery policy. Automatic completion wakeups are
diff --git a/packages/data-schemas/src/methods/conversation.spec.ts b/packages/data-schemas/src/methods/conversation.spec.ts
index 6c1b2eea3d..704327518d 100644
--- a/packages/data-schemas/src/methods/conversation.spec.ts
+++ b/packages/data-schemas/src/methods/conversation.spec.ts
@@ -6449,7 +6449,9 @@ describe('Conversation Operations', () => {
},
]);
- await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(1);
+ const activity = { found: false };
+ await expect(methods.expireLegacyAgentEventActorReceipts(now, 1, activity)).resolves.toBe(1);
+ expect(activity.found).toBe(true);
await expect(
Conversation.findOne({ conversationId: oldConversationId })
.select('+agentEventActorReconciliations')
@@ -6625,9 +6627,20 @@ describe('Conversation Operations', () => {
awaitTerminalHandling: true,
});
- /** The first raw page contains no expired receipt. The second contains
- * a protected receipt, and only the third reaches removable work. */
- await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(0);
+ /** The first raw page contains no expired receipt. It must still keep
+ * discovery active, without querying delivery protection for an empty ID set.
+ * The second is protected, and only the third reaches removable work. */
+ const activity = { found: false };
+ const protection = jest.spyOn(Delivery, 'find');
+ try {
+ await expect(methods.expireLegacyAgentEventActorReceipts(now, 1, activity)).resolves.toBe(
+ 0,
+ );
+ expect(activity.found).toBe(true);
+ expect(protection).not.toHaveBeenCalled();
+ } finally {
+ protection.mockRestore();
+ }
await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(0);
await expect(methods.expireLegacyAgentEventActorReceipts(now, 1)).resolves.toBe(1);
await expect(
diff --git a/packages/data-schemas/src/methods/conversation.ts b/packages/data-schemas/src/methods/conversation.ts
index f81e52fb11..71811f2f44 100644
--- a/packages/data-schemas/src/methods/conversation.ts
+++ b/packages/data-schemas/src/methods/conversation.ts
@@ -449,7 +449,11 @@ export interface ConversationMethods {
getAgentEventActorReconciliationStorageMetrics(
now: Date,
): Promise<AgentEventActorReconciliationStorageMetrics>;
- expireLegacyAgentEventActorReceipts(now: Date, limit?: number): Promise<number>;
+ expireLegacyAgentEventActorReceipts(
+ now: Date,
+ limit?: number,
+ activity?: { found: boolean },
+ ): Promise<number>;
reserveSubagentThread(input: {
user: string;
conversationId: string;
@@ -1782,7 +1786,11 @@ export function createConversationMethods(
/** Bounded mixed-version cleanup for terminal receipts embedded by older
* builds. New receipts expire through the delivery collection TTL index,
* but dormant legacy conversations need an independent retirement path. */
- async function expireLegacyAgentEventActorReceipts(now: Date, limit = 100): Promise<number> {
+ async function expireLegacyAgentEventActorReceipts(
+ now: Date,
+ limit = 100,
+ activity?: { found: boolean },
+ ): Promise<number> {
if (Number.isNaN(now.getTime())) {
throw new TypeError('now must be a valid date');
}
@@ -1821,6 +1829,9 @@ export function createConversationMethods(
return 0;
}
legacyReceiptExpiryCursor = candidates[candidates.length - 1]._id;
+ // A full page is not an empty sweep: later conversations may contain
+ // expired receipts. Keep scanning at base cadence until the page walk ends.
+ if (activity != null && candidates.length === boundedLimit) activity.found = true;
const expiredInvocationIds = [
...new Set(
candidates.flatMap((candidate) =>
@@ -1830,6 +1841,8 @@ export function createConversationMethods(
),
),
];
+ if (expiredInvocationIds.length === 0) return 0;
+ if (activity != null) activity.found = true;
const protectedInvocationIds = new Set(
await Delivery.find({
deliveryKey: { $in: expiredInvocationIds },
diff --git a/packages/data-schemas/src/methods/queuedTurn.spec.ts b/packages/data-schemas/src/methods/queuedTurn.spec.ts
index f4718dca8e..74c2f95f54 100644
--- a/packages/data-schemas/src/methods/queuedTurn.spec.ts
+++ b/packages/data-schemas/src/methods/queuedTurn.spec.ts
@@ -220,6 +220,31 @@ describe('agent queued turn methods', () => {
).resolves.toMatchObject({ status: 'queued', sequence: expect.any(Number) });
});
+ it('reports reservation-only recovery activity when deleted lanes yield no deliveries', async () => {
+ const first = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'lost-lane-first' }),
+ );
+ const second = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'lost-lane-second' }),
+ );
+ await Turn.updateMany(
+ { _id: { $in: [first.turn.queuedTurnId, second.turn.queuedTurnId] } },
+ {
+ $set: { status: 'reserving' },
+ $unset: { sequence: 1 },
+ },
+ );
+ await Sequence.deleteMany({});
+ for (let i = 0; i < 2; i++) {
+ const activity = { found: false };
+ expect(await methods.findQueuedTurnsNeedingDelivery(1, activity)).toEqual([]);
+ expect(activity.found).toBe(true);
+ }
+ const empty = { found: false };
+ expect(await methods.findQueuedTurnsNeedingDelivery(1, empty)).toEqual([]);
+ expect(empty.found).toBe(false);
+ });
+
it('caps each conversation at 100 active turns while preserving exact replay', async () => {
const inputs = Array.from({ length: 101 }, (_, index) =>
enqueueInput({ clientRequestId: `capacity-${index}`, text: `turn ${index}` }),
diff --git a/packages/data-schemas/src/methods/queuedTurn.ts b/packages/data-schemas/src/methods/queuedTurn.ts
index 22085ba0d9..f10bcb17ae 100644
--- a/packages/data-schemas/src/methods/queuedTurn.ts
+++ b/packages/data-schemas/src/methods/queuedTurn.ts
@@ -203,9 +203,13 @@ export interface AgentQueuedTurnMethods {
listAgentQueuedTurnReceipts: (
input: AgentQueuedTurnConversationScope & { clientRequestIds?: readonly string[] },
) => Promise<AgentQueuedTurnActiveRecord[]>;
- findQueuedTurnsNeedingDelivery: (limit?: number) => Promise<AgentQueuedTurnRecord[]>;
+ findQueuedTurnsNeedingDelivery: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<AgentQueuedTurnRecord[]>;
claimQueuedTurnsForAdmissionReconciliation: (
input: ClaimAgentQueuedTurnReconciliationInput,
+ activity?: { found: boolean },
) => Promise<AgentQueuedTurnRecord[]>;
deferAgentQueuedTurnAdmissionReconciliation: (
input: AgentQueuedTurnConversationScope & {
@@ -1381,7 +1385,10 @@ export function createAgentQueuedTurnMethods(
return { outcome: 'not_cancellable', turn: toRecord(current) };
}
- async function findQueuedTurnsNeedingDelivery(limit = 100): Promise<AgentQueuedTurnRecord[]> {
+ async function findQueuedTurnsNeedingDelivery(
+ limit = 100,
+ activity?: { found: boolean },
+ ): Promise<AgentQueuedTurnRecord[]> {
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1000) {
throw new TypeError('Agent queued turn recovery limit must be between 1 and 1000');
}
@@ -1390,6 +1397,7 @@ export function createAgentQueuedTurnMethods(
.sort({ createdAt: 1, _id: 1 })
.limit(limit)
.lean<IAgentQueuedTurn[]>();
+ if (activity != null && reservations.length > 0) activity.found = true;
for (const reservation of reservations) {
const scopeInput: AgentQueuedTurnConversationScope = {
user: reservation.user,
@@ -1460,6 +1468,7 @@ export function createAgentQueuedTurnMethods(
async function claimQueuedTurnsForAdmissionReconciliation(
input: ClaimAgentQueuedTurnReconciliationInput,
+ activity?: { found: boolean },
): Promise<AgentQueuedTurnRecord[]> {
const limit = input.limit ?? 100;
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1000) {
@@ -1519,6 +1528,7 @@ export function createAgentQueuedTurnMethods(
if (candidates.length === 0) {
return [];
}
+ if (activity != null) activity.found = true;
await Turn().updateMany(
{
...eligible,
diff --git a/packages/data-schemas/src/methods/triggerDelivery.spec.ts b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
index 1f0fcc74f6..c2b6110930 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.spec.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.spec.ts
@@ -1744,6 +1744,74 @@ describe('agent trigger delivery methods', () => {
});
});
+ it.each(['complete', 'dead', 'cleanup'] as const)(
+ 'signals maintenance after %s finalization fails without undoing root settlement',
+ async (mode) => {
+ const user = new mongoose.Types.ObjectId();
+ const coalesceUntil = new Date(Date.now() + 60_000);
+ const shared = {
+ user,
+ orderingKey: 'inline-recovery',
+ coalesceKey: 'inline-batch',
+ coalesceFrom: new Date(coalesceUntil.getTime() - 750),
+ coalesceUntil,
+ availableAt: coalesceUntil,
+ envelopeBytes: 128,
+ };
+ const root = await methods.enqueueAgentTriggerDelivery(enqueueInput(shared));
+ if (mode !== 'cleanup') await methods.enqueueAgentTriggerDelivery(enqueueInput(shared));
+ const claimed = await methods.claimNextAgentTriggerDelivery({
+ workerId: 'worker-1',
+ claimToken: 'inline-finalization-claim',
+ now: coalesceUntil,
+ leaseUntil: new Date(coalesceUntil.getTime() + 60_000),
+ });
+ expect(claimed?.id).toBe(root.delivery.id);
+ const input = {
+ id: claimed!.id,
+ workerId: 'worker-1',
+ claimToken: claimed!.claimToken!,
+ attempt: 1,
+ settledAt: coalesceUntil,
+ };
+ const failure =
+ mode === 'cleanup'
+ ? jest
+ .spyOn(LaneSequence, 'updateOne')
+ .mockRejectedValueOnce(new Error('cleanup interrupted'))
+ : jest
+ .spyOn(Delivery, 'updateMany')
+ .mockRejectedValueOnce(new Error('batch interrupted'));
+ const recovery = { required: false };
+ try {
+ const settled =
+ mode === 'dead'
+ ? await methods.deadLetterAgentTriggerDelivery(
+ { ...input, error: transientFailure({ retryable: false }) },
+ recovery,
+ )
+ : await methods.completeAgentTriggerDelivery(
+ { ...input, result: { accepted: true } },
+ recovery,
+ );
+ expect(settled).toBe(true);
+ expect(recovery.required).toBe(true);
+ } finally {
+ failure.mockRestore();
+ }
+ expect((await Delivery.findById(root.delivery.id).lean())?.status).toBe(
+ mode === 'dead' ? 'dead' : 'succeeded',
+ );
+ await methods.recoverAgentTriggerBatchReceipts();
+ await methods.reclaimInactiveAgentTriggerLanes();
+ const rows = await Delivery.find({ orderingKey: shared.orderingKey }).lean();
+ expect(rows.every((row) => row.status === (mode === 'dead' ? 'dead' : 'succeeded'))).toBe(
+ true,
+ );
+ if (mode !== 'dead') expect(await LaneSequence.findById(shared.orderingKey)).toBeNull();
+ },
+ );
+
it('recovers batch receipts and lane cleanup after root settlement was interrupted', async () => {
const user = new mongoose.Types.ObjectId();
const coalesceUntil = new Date(Date.now() + 60_000);
@@ -2244,6 +2312,9 @@ describe('agent trigger delivery methods', () => {
});
it('leaves staging unpublished while its durable user purge marker exists', async () => {
+ const emptyActivity = { found: false };
+ await expect(methods.recoverAgentTriggerLanePublications(1, emptyActivity)).resolves.toBe(0);
+ expect(emptyActivity.found).toBe(false);
const user = new mongoose.Types.ObjectId();
const orderingKey = 'purge-fenced-staging';
await UserPurge.create({ _id: user, fenceStartedAt: START, tenantId: 'tenant-1' });
@@ -2256,7 +2327,9 @@ describe('agent trigger delivery methods', () => {
stagingRecoveryAt: START,
});
- await expect(methods.recoverAgentTriggerLanePublications(1)).resolves.toBe(0);
+ const activity = { found: false };
+ await expect(methods.recoverAgentTriggerLanePublications(1, activity)).resolves.toBe(0);
+ expect(activity.found).toBe(true);
await expect(Delivery.findById(staged._id).lean()).resolves.toMatchObject({
status: 'staging',
laneSequence: 0,
diff --git a/packages/data-schemas/src/methods/triggerDelivery.ts b/packages/data-schemas/src/methods/triggerDelivery.ts
index 0137a6dd43..e3fe4c2d0f 100644
--- a/packages/data-schemas/src/methods/triggerDelivery.ts
+++ b/packages/data-schemas/src/methods/triggerDelivery.ts
@@ -281,18 +281,22 @@ export interface AgentTriggerDeliveryMethods {
handling?: AgentTriggerHandlingState;
awaitTerminalHandling?: true;
},
+ recovery?: { required: boolean },
+ ) => Promise<boolean>;
+ retireAgentTriggerDelivery: (
+ input: {
+ deliveryKey: string;
+ sourceId: string;
+ settledAt: Date;
+ reason: string;
+ onlyIfUnclaimed?: boolean;
+ onlyIfDead?: boolean;
+ /** Accept transport success without a terminal handling receipt, unless the
+ * delivery explicitly keeps its lane open for terminal handling. */
+ allowSucceeded?: boolean;
+ },
+ recovery?: { required: boolean },
) => Promise<boolean>;
- retireAgentTriggerDelivery: (input: {
- deliveryKey: string;
- sourceId: string;
- settledAt: Date;
- reason: string;
- onlyIfUnclaimed?: boolean;
- onlyIfDead?: boolean;
- /** Accept transport success without a terminal handling receipt, unless the
- * delivery explicitly keeps its lane open for terminal handling. */
- allowSucceeded?: boolean;
- }) => Promise<boolean>;
renewAgentTriggerDeliveryProducerLease: (input: {
deliveryKey: string;
sourceId: string;
@@ -385,6 +389,7 @@ export interface AgentTriggerDeliveryMethods {
settledAt: Date;
receiptRetryAt?: Date;
},
+ recovery?: { required: boolean },
) => Promise<boolean>;
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;
getAgentTriggerDeliveryStatus: (
@@ -402,9 +407,18 @@ export interface AgentTriggerDeliveryMethods {
user: string | Types.ObjectId,
now: Date,
) => Promise<number>;
- recoverAgentTriggerLanePublications: (limit?: number) => Promise<number>;
- recoverAgentTriggerBatchReceipts: (limit?: number) => Promise<number>;
- reclaimInactiveAgentTriggerLanes: (limit?: number) => Promise<number>;
+ recoverAgentTriggerLanePublications: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
+ recoverAgentTriggerBatchReceipts: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
+ reclaimInactiveAgentTriggerLanes: (
+ limit?: number,
+ activity?: { found: boolean },
+ ) => Promise<number>;
prepareAgentTriggerUserPurge: (
user: string | Types.ObjectId,
fenceStartedAt: Date,
@@ -414,7 +428,7 @@ export interface AgentTriggerDeliveryMethods {
user: string | Types.ObjectId,
fenceStartedAt: Date,
) => Promise<boolean>;
- recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
+ recoverAgentTriggerUserPurges: (limit?: number, activity?: { found: boolean }) => Promise<number>;
deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;
eraseAgentTriggerDeliveryConversationResults: (
user: string | Types.ObjectId,
@@ -1092,6 +1106,7 @@ export function createAgentTriggerDeliveryMethods(
/** Repairs abandoned reservations and staging rows left by crashed writers. */
async function recoverAgentTriggerLanePublications(
limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ activity?: { found: boolean },
): Promise<number> {
if (!Number.isSafeInteger(limit) || limit <= 0) {
throw new TypeError('Agent trigger lane recovery limit must be a positive integer');
@@ -1105,6 +1120,7 @@ export function createAgentTriggerDeliveryMethods(
.sort({ publisherStartedAt: 1, _id: 1 })
.limit(boundedLimit)
.lean<IAgentTriggerLaneSequence[]>();
+ if (activity != null && lanes.length > 0) activity.found = true;
let recovered = 0;
const recoveryCursor = new Date();
for (const lane of lanes) {
@@ -1175,6 +1191,9 @@ export function createAgentTriggerDeliveryMethods(
.limit(remaining)
.lean<IAgentTriggerDelivery[]>()
: [];
+ if (activity != null && (legacyStaged.length > 0 || indexedStaged.length > 0)) {
+ activity.found = true;
+ }
const staged = [
...legacyStaged.map((delivery) => ({
...delivery,
@@ -1293,6 +1312,7 @@ export function createAgentTriggerDeliveryMethods(
/** Bounds high-cardinality ordering metadata after the final retained job settles. */
async function reclaimInactiveAgentTriggerLanes(
limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ activity?: { found: boolean },
): Promise<number> {
if (!Number.isSafeInteger(limit) || limit <= 0) {
throw new TypeError('Agent trigger lane reclamation limit must be a positive integer');
@@ -1304,6 +1324,7 @@ export function createAgentTriggerDeliveryMethods(
.limit(boundedLimit)
.select('_id orderingKey laneCleanupPendingAt')
.lean<Array<Pick<IAgentTriggerDelivery, '_id' | 'orderingKey' | 'laneCleanupPendingAt'>>>();
+ if (activity != null && pendingCleanup.length > 0) activity.found = true;
let reclaimed = 0;
for (const delivery of pendingCleanup) {
if (await fulfillLaneCleanupRequest(delivery)) {
@@ -1321,6 +1342,7 @@ export function createAgentTriggerDeliveryMethods(
.limit(remaining)
.select('_id')
.lean<Array<Pick<IAgentTriggerLaneSequence, '_id'>>>();
+ if (activity != null && lanes.length > 0) activity.found = true;
for (const lane of lanes) {
if (await reclaimLaneIfInactive(lane._id)) {
reclaimed += 1;
@@ -1760,6 +1782,7 @@ export function createAgentTriggerDeliveryMethods(
async function recoverAgentTriggerBatchReceipts(
limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ activity?: { found: boolean },
): Promise<number> {
if (!Number.isSafeInteger(limit) || limit <= 0) {
throw new TypeError('Agent trigger batch recovery limit must be a positive integer');
@@ -1773,6 +1796,7 @@ export function createAgentTriggerDeliveryMethods(
.sort({ settledAt: 1, _id: 1 })
.limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
.lean<IAgentTriggerDelivery[]>();
+ if (activity != null && roots.length > 0) activity.found = true;
let recovered = 0;
for (const root of roots) {
if (root.settledAt == null || (root.status !== 'succeeded' && root.status !== 'dead')) {
@@ -1970,6 +1994,7 @@ export function createAgentTriggerDeliveryMethods(
handling?: AgentTriggerHandlingState;
awaitTerminalHandling?: true;
},
+ recovery?: { required: boolean },
): Promise<boolean> {
const awaitsTerminalHandling =
input.awaitTerminalHandling === true && input.handling?.status === 'started';
@@ -2050,6 +2075,7 @@ export function createAgentTriggerDeliveryMethods(
});
await fulfillLaneCleanupRequest(completed);
} catch (error) {
+ if (recovery != null) recovery.required = true;
// Root success is authoritative. Maintenance retries both constituent
// receipt settlement and the existing durable lane-cleanup marker.
logger.warn('[agent-triggers] failed to finalize a completed trigger batch', {
@@ -2064,17 +2090,20 @@ export function createAgentTriggerDeliveryMethods(
* that the result will never become dispatchable. This transition is keyed
* by the immutable delivery identity rather than a worker lease so the
* producer can unblock the lane even while a resolver is deferring it. */
- async function retireAgentTriggerDelivery(input: {
- deliveryKey: string;
- sourceId: string;
- settledAt: Date;
- reason: string;
- onlyIfUnclaimed?: boolean;
- onlyIfDead?: boolean;
- /** Accept transport success without a terminal handling receipt, unless the
- * delivery explicitly keeps its lane open for terminal handling. */
- allowSucceeded?: boolean;
- }): Promise<boolean> {
+ async function retireAgentTriggerDelivery(
+ input: {
+ deliveryKey: string;
+ sourceId: string;
+ settledAt: Date;
+ reason: string;
+ onlyIfUnclaimed?: boolean;
+ onlyIfDead?: boolean;
+ /** Accept transport success without a terminal handling receipt, unless the
+ * delivery explicitly keeps its lane open for terminal handling. */
+ allowSucceeded?: boolean;
+ },
+ recovery?: { required: boolean },
+ ): Promise<boolean> {
if (
input.deliveryKey.length === 0 ||
input.deliveryKey.length > 256 ||
@@ -2151,6 +2180,7 @@ export function createAgentTriggerDeliveryMethods(
try {
await fulfillLaneCleanupRequest(retired);
} catch (error) {
+ if (recovery != null) recovery.required = true;
logger.warn('[agent-triggers] failed to finalize a retired internal delivery', {
deliveryKey: input.deliveryKey,
error: error instanceof Error ? error.message : String(error),
@@ -3509,6 +3539,7 @@ export function createAgentTriggerDeliveryMethods(
settledAt: Date;
receiptRetryAt?: Date;
},
+ recovery?: { required: boolean },
): Promise<boolean> {
const error = normalizeFailure(input.error);
if (
@@ -3603,6 +3634,7 @@ export function createAgentTriggerDeliveryMethods(
error,
});
} catch (settlementError) {
+ if (recovery != null) recovery.required = true;
logger.warn('[agent-triggers] failed to settle batch dead-letter receipts', {
deliveryId: String(dead._id),
error: settlementError instanceof Error ? settlementError.message : String(settlementError),
@@ -3881,6 +3913,7 @@ export function createAgentTriggerDeliveryMethods(
/** Recovers cleanup markers whose users are gone; active-user markers are never destructive. */
async function recoverAgentTriggerUserPurges(
limit = DEFAULT_PURGE_RECOVERY_LIMIT,
+ activity?: { found: boolean },
): Promise<number> {
if (!Number.isSafeInteger(limit) || limit <= 0) {
throw new TypeError('Agent trigger purge recovery limit must be a positive integer');
@@ -3893,6 +3926,7 @@ export function createAgentTriggerDeliveryMethods(
.sort({ backgroundToolResultDeletionPendingAt: 1, _id: 1 })
.limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
.lean<IAgentTriggerDelivery[]>();
+ if (activity != null && pending.length > 0) activity.found = true;
for (const row of pending) {
const conversationId = (row.envelope as { target?: { conversationId?: string } }).target
?.conversationId;
@@ -3915,6 +3949,7 @@ export function createAgentTriggerDeliveryMethods(
.sort({ updatedAt: 1, _id: 1 })
.limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
.lean<IAgentTriggerUserPurge[]>();
+ if (activity != null && markers.length > 0) activity.found = true;
let recovered = 0;
for (const marker of markers) {
const user = await mongoose.models.User.findById(marker._id)