Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛬 fix: Settle Background Tool Results Before the Server Shuts Down (#16365)

Danny Avilac606e5065a65
diff --git a/api/server/experimental.js b/api/server/experimental.js
index 45332738d4..276beb8ed0 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -43,8 +43,8 @@ const {
configureServerTimeouts,
setupGracefulShutdown,
registerShutdownTask,
- getRemainingShutdownMs,
- getShutdownElapsedMs,
+ getClusterShutdownBudgetMs,
+ registerBackgroundTaskShutdown,
configureMessageFilterRegexValidator,
configureFileConfigRegexEngine,
configureAgentEventRuntime,
@@ -386,24 +386,18 @@ if (cluster.isMaster) {
* after signalling. Measure against that, and hold back a reserve for the tasks after this
* one. Abandoning an unrecorded drain fences the next generation permanently. */
const CLUSTER_TEARDOWN_RESERVE_MS = 3_000;
+ const clusterShutdownBudgetMs = () =>
+ getClusterShutdownBudgetMs({
+ deadlineAt: clusterShutdownDeadlineAt,
+ forceExitMs: CLUSTER_FORCE_EXIT_MS,
+ });
const destroyGenerationJobManager = () => {
- const remaining = getRemainingShutdownMs();
- const elapsed = getShutdownElapsedMs();
- if (remaining == null || elapsed == null) {
+ const budgetMs = clusterShutdownBudgetMs();
+ if (budgetMs == null) {
return GenerationJobManager.destroy();
}
- /** Prefer the deadline the primary actually set. The elapsed-based estimate starts
- * counting only when this worker's signal handler ran, which lags the primary's timer
- * by however long the event loop was blocked. */
- const primaryRemaining =
- clusterShutdownDeadlineAt != null
- ? clusterShutdownDeadlineAt - Date.now()
- : CLUSTER_FORCE_EXIT_MS - elapsed;
return GenerationJobManager.destroy({
- settlementBudgetMs: Math.max(
- 0,
- Math.min(remaining, primaryRemaining) - CLUSTER_TEARDOWN_RESERVE_MS,
- ),
+ settlementBudgetMs: Math.max(0, budgetMs - CLUSTER_TEARDOWN_RESERVE_MS),
});
};
// Tear down stream resources before shared caches and telemetry exporters shut down.
@@ -508,6 +502,10 @@ if (cluster.isMaster) {
// principal) still merges DB `__base__` overrides, which must not drive which hook
// modules load in every worker (matches api/server/index.js's baseOnly usage).
const baseAppConfig = await getAppConfig({ baseOnly: true });
+ registerBackgroundTaskShutdown({
+ interruptGraceMs: baseAppConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs,
+ getBudgetMs: clusterShutdownBudgetMs,
+ });
configureAgentEventRuntime(baseAppConfig?.endpoints?.agents?.eventDriven);
const toolApproval = baseAppConfig?.endpoints?.agents?.toolApproval;
await loadToolApprovalHooks(toolApproval?.enabled ? toolApproval.hooks : undefined, {
diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.js
index b1f05d4663..30d2878a09 100644
--- a/api/server/experimental.spec.js
+++ b/api/server/experimental.spec.js
@@ -23,6 +23,18 @@ describe('Experimental server configuration', () => {
expect(source).toMatch(/if \(shuttingDown\) \{[\s\S]*?return;[\s\S]*?Starting a new worker/);
});
+ it("drains background tasks within the primary's cluster shutdown deadline", () => {
+ expect(source).toMatch(
+ /registerBackgroundTaskShutdown\(\{[\s\S]*?getBudgetMs: clusterShutdownBudgetMs,[\s\S]*?\}\);/,
+ );
+ expect(source).toMatch(
+ /const clusterShutdownBudgetMs = \(\) =>\s*getClusterShutdownBudgetMs\(\{\s*deadlineAt: clusterShutdownDeadlineAt,\s*forceExitMs: CLUSTER_FORCE_EXIT_MS,\s*\}\);/,
+ );
+ expect(source).toMatch(
+ /const destroyGenerationJobManager = \(\) => \{\s*const budgetMs = clusterShutdownBudgetMs\(\);/,
+ );
+ });
+
it('starts approval expiry after installing the scheduled-run callback', () => {
const handlerIndex = source.indexOf(
'GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);',
diff --git a/api/server/index.js b/api/server/index.js
index 1d31f63b26..fa3412b0f7 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -48,6 +48,7 @@ const {
requestContextMiddleware,
registerShutdownTask,
getRemainingShutdownMs,
+ registerBackgroundTaskShutdown,
configureServerTimeouts,
setupGracefulShutdown,
updateInterfacePermissions,
@@ -231,6 +232,9 @@ const startServer = async () => {
logger.error('[sweepOrphanedPreviews] Background sweep failed:', err);
});
const appConfig = await getAppConfig({ baseOnly: true });
+ registerBackgroundTaskShutdown({
+ interruptGraceMs: appConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs,
+ });
configureAgentEventRuntime(appConfig?.endpoints?.agents?.eventDriven);
warnOnUnreachableDeliveryPaths(appConfig);
initializeFileStorage(appConfig);
diff --git a/api/server/index.spec.js b/api/server/index.spec.js
index e1b565dcc3..5f5a54c475 100644
--- a/api/server/index.spec.js
+++ b/api/server/index.spec.js
@@ -188,6 +188,17 @@ describe('Startup readiness wiring', () => {
expect(shutdownRegistrationIndex).toBeLessThan(listenIndex);
});
+ it('registers background task draining with the graceful shutdown coordinator', () => {
+ const registrationIndex = source.indexOf('registerBackgroundTaskShutdown({');
+ const listenIndex = source.indexOf('const server = app.listen');
+
+ expect(registrationIndex).toBeGreaterThan(-1);
+ expect(registrationIndex).toBeLessThan(listenIndex);
+ expect(source).toContain(
+ 'interruptGraceMs: appConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs',
+ );
+ });
+
it('configures HTTP timeouts before graceful shutdown handling', () => {
const listenIndex = source.indexOf('const server = app.listen');
const timeoutConfigIndex = source.indexOf('configureServerTimeouts(server);');
diff --git a/packages/api/src/agents/background.shutdown.spec.ts b/packages/api/src/agents/background.shutdown.spec.ts
new file mode 100644
index 0000000000..83d1684b02
--- /dev/null
+++ b/packages/api/src/agents/background.shutdown.spec.ts
@@ -0,0 +1,281 @@
+import { AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT } from 'librechat-data-provider';
+import type { BackgroundTask, BackgroundTaskShutdownHandle } from './background';
+import {
+ BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+ BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,
+ BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS,
+} from './backgroundCompletion';
+import {
+ BackgroundTaskRegistryClass,
+ buildBackgroundCapacityContent,
+ registerBackgroundTaskShutdown,
+} from './background';
+import * as shutdown from '~/app/shutdown';
+
+const REASON = 'server shutting down';
+
+function createTask(registry: BackgroundTaskRegistryClass, toolCallId: string): BackgroundTask {
+ const created = registry.create({
+ userId: 'shutdown-owner',
+ conversationId: 'shutdown-conversation',
+ toolCallId,
+ toolName: 'bash_tool',
+ });
+ if ('atCapacity' in created) {
+ throw new Error('unexpected capacity');
+ }
+ return created.task;
+}
+
+interface ControlledHandle {
+ handle: BackgroundTaskShutdownHandle;
+ interrupt: jest.Mock;
+ flush: jest.Mock;
+ settle: () => void;
+}
+
+/** A handle whose settlement the test drives; `onInterrupt`/`flushResult` model the tool. */
+function controlledHandle(
+ options: { onInterrupt?: 'settle' | 'ignore'; flushResult?: 'settle' | 'hang' } = {},
+): ControlledHandle {
+ let settle: () => void = () => undefined;
+ const settled = new Promise<void>((resolve) => {
+ settle = resolve;
+ });
+ const interrupt = jest.fn(() => {
+ if (options.onInterrupt === 'settle') {
+ settle();
+ }
+ });
+ const flush = jest.fn(() => {
+ if (options.flushResult === 'hang') {
+ return new Promise<void>(() => undefined);
+ }
+ settle();
+ return Promise.resolve();
+ });
+ return { handle: { settled, interrupt, flush }, interrupt, flush, settle: () => settle() };
+}
+
+const drainOptions = (overrides: { deadlineMs?: number; interruptGraceMs?: number } = {}) => ({
+ deadlineAt: Date.now() + (overrides.deadlineMs ?? 400),
+ interruptGraceMs: overrides.interruptGraceMs ?? 100,
+ flushReserveMs: 50,
+ reason: REASON,
+});
+
+describe('background task shutdown', () => {
+ describe('admission', () => {
+ it('refuses new tasks once admission closes, but still resolves replays and reserved permits', () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const existing = createTask(registry, 'before-shutdown');
+ const permitAdmission = registry.reserveCapacity({
+ userId: 'shutdown-owner',
+ conversationId: 'shutdown-conversation',
+ toolCallId: 'reserved-before-shutdown',
+ });
+ if (!('permit' in permitAdmission)) {
+ throw new Error('expected a capacity permit');
+ }
+
+ registry.closeAdmission();
+
+ expect(registry.isAdmissionClosed()).toBe(true);
+ expect(
+ registry.create({
+ userId: 'shutdown-owner',
+ conversationId: 'shutdown-conversation',
+ toolCallId: 'after-shutdown',
+ toolName: 'bash_tool',
+ }),
+ ).toEqual({ atCapacity: true, scope: 'shutting_down' });
+ expect(
+ registry.reserveCapacity({
+ userId: 'shutdown-owner',
+ conversationId: 'shutdown-conversation',
+ toolCallId: 'after-shutdown',
+ }),
+ ).toEqual({ atCapacity: true, scope: 'shutting_down' });
+ expect(
+ registry.create({
+ userId: 'shutdown-owner',
+ conversationId: 'shutdown-conversation',
+ toolCallId: 'before-shutdown',
+ toolName: 'bash_tool',
+ }),
+ ).toEqual({ task: existing, isNew: false });
+ expect(
+ registry.create({
+ userId: 'shutdown-owner',
+ conversationId: 'shutdown-conversation',
+ toolCallId: 'reserved-before-shutdown',
+ toolName: 'bash_tool',
+ capacityPermit: permitAdmission.permit,
+ }),
+ ).toMatchObject({ isNew: true });
+ });
+
+ it('tells the model why a background dispatch was refused', () => {
+ const content = JSON.parse(buildBackgroundCapacityContent('bash_tool', 'shutting_down'));
+ expect(content).toEqual({
+ status: 'rejected',
+ tool: 'bash_tool',
+ scope: 'shutting_down',
+ message: expect.stringContaining('shutting down'),
+ });
+ });
+ });
+
+ describe('drainForShutdown', () => {
+ it('returns immediately and closes admission when nothing is tracked', async () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const startedAt = Date.now();
+
+ const summary = await registry.drainForShutdown(drainOptions({ deadlineMs: 5_000 }));
+
+ expect(summary).toEqual({ tracked: 0, interrupted: 0, flushed: 0, unsettled: 0 });
+ expect(Date.now() - startedAt).toBeLessThan(1_000);
+ expect(registry.isAdmissionClosed()).toBe(true);
+ });
+
+ it('waits for tasks that settle on their own without interrupting them', async () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const task = createTask(registry, 'settles-alone');
+ const controlled = controlledHandle();
+ registry.trackShutdown(task, controlled.handle);
+ setTimeout(controlled.settle, 20);
+
+ const summary = await registry.drainForShutdown(drainOptions());
+
+ expect(summary).toEqual({ tracked: 1, interrupted: 0, flushed: 0, unsettled: 0 });
+ expect(controlled.interrupt).not.toHaveBeenCalled();
+ expect(controlled.flush).not.toHaveBeenCalled();
+ });
+
+ it('interrupts running tasks, then flushes only those that did not settle after the abort', async () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const cooperative = controlledHandle({ onInterrupt: 'settle' });
+ const stubborn = controlledHandle({ onInterrupt: 'ignore' });
+ registry.trackShutdown(createTask(registry, 'cooperative'), cooperative.handle);
+ registry.trackShutdown(createTask(registry, 'stubborn'), stubborn.handle);
+
+ const summary = await registry.drainForShutdown(drainOptions());
+
+ expect(cooperative.interrupt).toHaveBeenCalledWith(REASON);
+ expect(stubborn.interrupt).toHaveBeenCalledWith(REASON);
+ expect(cooperative.flush).not.toHaveBeenCalled();
+ expect(stubborn.flush).toHaveBeenCalledWith(REASON);
+ expect(summary).toEqual({ tracked: 2, interrupted: 2, flushed: 1, unsettled: 0 });
+ });
+
+ it('flushes a settled task whose result is not durable yet without aborting it', async () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const task = createTask(registry, 'settled-not-durable');
+ const controlled = controlledHandle();
+ registry.trackShutdown(task, controlled.handle);
+ registry.complete('shutdown-owner', 'shutdown-conversation', task.id, { content: 'done' });
+
+ const summary = await registry.drainForShutdown(drainOptions());
+
+ expect(controlled.interrupt).not.toHaveBeenCalled();
+ expect(controlled.flush).toHaveBeenCalledWith(REASON);
+ expect(summary).toEqual({ tracked: 1, interrupted: 0, flushed: 1, unsettled: 0 });
+ });
+
+ it('stops at the deadline and reports results it could not confirm', async () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const hanging = controlledHandle({ flushResult: 'hang' });
+ registry.trackShutdown(createTask(registry, 'hanging'), hanging.handle);
+ const startedAt = Date.now();
+
+ const summary = await registry.drainForShutdown(drainOptions({ deadlineMs: 300 }));
+
+ expect(Date.now() - startedAt).toBeLessThan(2_000);
+ expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 1, unsettled: 1 });
+ });
+
+ it('stops tracking a task once its result is durable', async () => {
+ const registry = new BackgroundTaskRegistryClass();
+ const controlled = controlledHandle();
+ registry.trackShutdown(createTask(registry, 'released'), controlled.handle);
+
+ controlled.settle();
+ await controlled.handle.settled;
+ await Promise.resolve();
+
+ const summary = await registry.drainForShutdown(drainOptions());
+ expect(summary.tracked).toBe(0);
+ });
+ });
+
+ describe('registerBackgroundTaskShutdown', () => {
+ type RegisteredTask = [string, () => void | Promise<void>, shutdown.ShutdownTaskOptions?];
+
+ function captureRegistrations(): RegisteredTask[] {
+ const registered: RegisteredTask[] = [];
+ jest.spyOn(shutdown, 'registerShutdownTask').mockImplementation((name, fn, options) => {
+ registered.push([name, fn, options]);
+ });
+ return registered;
+ }
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it('closes admission before the HTTP drain and settles tasks after generations finalize', async () => {
+ const registered = captureRegistrations();
+ const registry = new BackgroundTaskRegistryClass();
+ const drain = jest.spyOn(registry, 'drainForShutdown');
+
+ registerBackgroundTaskShutdown({ registry, getBudgetMs: () => 10_000 });
+
+ expect(registered.map(([name, , options]) => [name, options])).toEqual([
+ ['background task admission', { phase: 'pre-drain', priority: 110 }],
+ ['background tasks', { priority: 95 }],
+ ]);
+ expect(registered[0][2]?.priority).toBeGreaterThan(100);
+ registered[0][1]();
+ expect(registry.isAdmissionClosed()).toBe(true);
+
+ const startedAt = Date.now();
+ await registered[1][1]();
+ expect(drain).toHaveBeenCalledWith({
+ deadlineAt: expect.any(Number),
+ interruptGraceMs: AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT,
+ flushReserveMs: BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,
+ reason: BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+ });
+ const { deadlineAt } = drain.mock.calls[0][0];
+ const expected = startedAt + 10_000 - BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS;
+ expect(Math.abs(deadlineAt - expected)).toBeLessThan(1_000);
+ });
+
+ it('uses the configured interrupt grace', async () => {
+ const registered = captureRegistrations();
+ const registry = new BackgroundTaskRegistryClass();
+ const drain = jest.spyOn(registry, 'drainForShutdown');
+
+ registerBackgroundTaskShutdown({
+ registry,
+ interruptGraceMs: 1_234,
+ getBudgetMs: () => 5_000,
+ });
+ await registered[1][1]();
+
+ expect(drain.mock.calls[0][0].interruptGraceMs).toBe(1_234);
+ });
+
+ it('only closes admission when no shutdown budget is known', async () => {
+ const registered = captureRegistrations();
+ const registry = new BackgroundTaskRegistryClass();
+ const drain = jest.spyOn(registry, 'drainForShutdown');
+
+ registerBackgroundTaskShutdown({ registry, getBudgetMs: () => null });
+ await registered[1][1]();
+
+ expect(drain).not.toHaveBeenCalled();
+ expect(registry.isAdmissionClosed()).toBe(true);
+ });
+ });
+});
diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts
index c95cb89524..700a33e730 100644
--- a/packages/api/src/agents/background.ts
+++ b/packages/api/src/agents/background.ts
@@ -36,7 +36,12 @@
import { logger } from '@librechat/data-schemas';
import { createHash, randomUUID } from 'node:crypto';
import { Constants as AgentConstants } from '@librechat/agents';
-import { Tools, Constants, imageGenTools } from 'librechat-data-provider';
+import {
+ Tools,
+ Constants,
+ imageGenTools,
+ AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT,
+} from 'librechat-data-provider';
import type {
LCTool,
LCToolRegistry,
@@ -57,6 +62,9 @@ import type { BackgroundToolResultState } from './harvest';
import type { CapabilityToolNames } from './selection';
import {
BACKGROUND_TASK_TIMEOUT_MS,
+ BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+ BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,
+ BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS,
type PendingBackgroundCompletion,
type BackgroundToolDeadClaimRecovery,
type BackgroundToolWakeupAdmission,
@@ -75,6 +83,7 @@ import {
synthesizeSelectionToolOptions,
} from './selection';
import { SUBAGENT_WAKEUP_GUIDANCE, agentUsesSubagentCompletionWakeups } from './subagentDelivery';
+import { registerShutdownTask, getRemainingShutdownMs } from '~/app/shutdown';
import { SubagentTaskOwnerUnavailableError } from './subagentTaskRouting';
import { SET_MEMORY_TOOL_NAME, DELETE_MEMORY_TOOL_NAME } from './memory';
import { ASK_USER_QUESTION_TOOL_NAME } from './hitl/askUserQuestionTool';
@@ -711,13 +720,53 @@ export type BackgroundTaskCapacityScope =
| 'user_running'
| 'user_retention'
| 'global_running'
- | 'global_retention';
+ | 'global_retention'
+ | 'shutting_down';
type BackgroundTaskCapacityRejection = {
atCapacity: true;
scope: BackgroundTaskCapacityScope;
};
+/** Process-local controls a running task registers so graceful shutdown can settle it. */
+export interface BackgroundTaskShutdownHandle {
+ /** Resolves once the task's result is durable, or once the task finished when it has
+ * no durable delivery. Never rejects. */
+ settled: Promise<void>;
+ /** Aborts the live invocation because the server is shutting down. */
+ interrupt: (reason: string) => void;
+ /** Stores the best result available now: the task's own result when it has settled,
+ * otherwise an interrupted failure. */
+ flush: (reason: string) => Promise<void>;
+}
+
+export interface BackgroundTaskDrainSummary {
+ /** Tasks with an unsettled result when the drain began. */
+ tracked: number;
+ /** Tasks still running when the drain aborted them. */
+ interrupted: number;
+ /** Tasks whose result was still not durable, so the drain stored one. */
+ flushed: number;
+ /** Tasks whose durable result was still unconfirmed at the deadline. */
+ unsettled: number;
+}
+
+/** Resolves when `promise` settles or at `deadlineAt`, whichever comes first. */
+function waitUntil(promise: Promise<unknown>, deadlineAt: number): Promise<void> {
+ const remainingMs = deadlineAt - Date.now();
+ if (remainingMs <= 0) {
+ return Promise.resolve();
+ }
+ return new Promise<void>((resolve) => {
+ const timer = setTimeout(resolve, remainingMs);
+ const finish = (): void => {
+ clearTimeout(timer);
+ resolve();
+ };
+ promise.then(finish, finish);
+ });
+}
+
const COMPLETED_TASK_TTL_MS = 60 * 60 * 1000;
const IDLE_BUCKET_TTL_MS = 6 * 60 * 60 * 1000;
const MAX_RUNNING_PER_BUCKET = 10;
@@ -829,6 +878,9 @@ export class BackgroundTaskRegistryClass {
/** Live invocation controls are intentionally process-local and are never
* exposed through task snapshots or durable receipts. */
private readonly cancellationRequests = new WeakMap<BackgroundTask, () => boolean | void>();
+ /** Tasks whose result is not yet durable, for the graceful-shutdown drain. */
+ private readonly shutdownHandles = new Map<BackgroundTask, BackgroundTaskShutdownHandle>();
+ private admissionClosed = false;
private lastGlobalSweepAt = 0;
private key(userId: string, conversationId: string): string {
@@ -1171,6 +1223,9 @@ export class BackgroundTaskRegistryClass {
if (existing != null) {
return { task: existing, isNew: false };
}
+ if (this.admissionClosed) {
+ return { atCapacity: true, scope: 'shutting_down' };
+ }
if (
existingBucket != null &&
this.runningCount(existingBucket) + existingBucket.capacityPermits.size >=
@@ -1261,6 +1316,11 @@ export class BackgroundTaskRegistryClass {
}
}
+ /** A permit already reserved launch authority before shutdown began, so it may still land. */
+ if (this.admissionClosed && params.capacityPermit == null) {
+ return { atCapacity: true, scope: 'shutting_down' };
+ }
+
if (params.capacityPermit != null) {
if (existingBucket == null) {
throw new Error('Background task capacity permit is stale');
@@ -1348,6 +1408,85 @@ export class BackgroundTaskRegistryClass {
return { status: 'requested', task };
}
+ /** Refuses new background tasks; the server is shutting down. Replays still resolve. */
+ closeAdmission(): void {
+ this.admissionClosed = true;
+ }
+
+ isAdmissionClosed(): boolean {
+ return this.admissionClosed;
+ }
+
+ /** Registers a new task's shutdown controls until its result is durable. */
+ trackShutdown(task: BackgroundTask, handle: BackgroundTaskShutdownHandle): void {
+ this.shutdownHandles.set(task, handle);
+ const release = (): void => {
+ if (this.shutdownHandles.get(task) === handle) {
+ this.shutdownHandles.delete(task);
+ }
+ };
+ handle.settled.then(release, release);
+ }
+
+ /**
+ * Settles every tracked task before the process exits. Tasks first get whatever time
+ * is left before `interruptGraceMs` and the flush reserve; tasks still running are then
+ * aborted and get `interruptGraceMs` to settle on their own; the rest have a durable
+ * result written for them. Without this, a restart abandons them and their automatic
+ * deliveries dead-letter once the producer lease expires.
+ */
+ async drainForShutdown(options: {
+ deadlineAt: number;
+ interruptGraceMs: number;
+ flushReserveMs: number;
+ reason: string;
+ }): Promise<BackgroundTaskDrainSummary> {
+ this.closeAdmission();
+ const tracked = this.shutdownHandles.size;
+ if (tracked === 0) {
+ return { tracked: 0, interrupted: 0, flushed: 0, unsettled: 0 };
+ }
+ const flushAt = options.deadlineAt - options.flushReserveMs;
+ const interruptAt = flushAt - options.interruptGraceMs;
+
+ await this.waitForShutdownHandles(interruptAt);
+ let interrupted = 0;
+ for (const [task, handle] of this.shutdownHandles) {
+ if (task.status === 'running') {
+ interrupted++;
+ handle.interrupt(options.reason);
+ }
+ }
+
+ await this.waitForShutdownHandles(flushAt);
+ const remaining = [...this.shutdownHandles.values()];
+ let durable = 0;
+ await waitUntil(
+ Promise.allSettled(
+ remaining.map((handle) =>
+ handle.flush(options.reason).then(() => {
+ durable++;
+ }),
+ ),
+ ),
+ options.deadlineAt,
+ );
+ return {
+ tracked,
+ interrupted,
+ flushed: remaining.length,
+ unsettled: remaining.length - durable,
+ };
+ }
+
+ private waitForShutdownHandles(deadlineAt: number): Promise<void> {
+ const pending = [...this.shutdownHandles.values()].map((handle) => handle.settled);
+ if (pending.length === 0) {
+ return Promise.resolve();
+ }
+ return waitUntil(Promise.all(pending), deadlineAt);
+ }
+
private update(
userId: string,
conversationId: string,
@@ -1795,6 +1934,57 @@ export class BackgroundTaskRegistryClass {
export const backgroundTaskRegistry: BackgroundTaskRegistryClass =
new BackgroundTaskRegistryClass();
+export interface BackgroundTaskShutdownOptions {
+ /** `endpoints.agents.backgroundTasks.shutdownInterruptGraceMs`. */
+ interruptGraceMs?: number;
+ /** Shutdown time left for the drain; defaults to the local coordinator's remaining budget.
+ * A clustered worker passes its primary's tighter deadline. */
+ getBudgetMs?: () => number | null;
+ registry?: BackgroundTaskRegistryClass;
+}
+
+/**
+ * Registers the graceful-shutdown steps for background tools: stop admitting new tasks
+ * while the HTTP server drains, then, after generations are finalized, settle every task
+ * whose result is not yet durable.
+ */
+export function registerBackgroundTaskShutdown(options: BackgroundTaskShutdownOptions = {}): void {
+ const registry = options.registry ?? backgroundTaskRegistry;
+ const getBudgetMs = options.getBudgetMs ?? getRemainingShutdownMs;
+ registerShutdownTask('background task admission', () => registry.closeAdmission(), {
+ phase: 'pre-drain',
+ priority: 110,
+ });
+ registerShutdownTask(
+ 'background tasks',
+ async () => {
+ const budgetMs = getBudgetMs();
+ if (budgetMs == null) {
+ registry.closeAdmission();
+ return;
+ }
+ const summary = await registry.drainForShutdown({
+ deadlineAt: Date.now() + Math.max(0, budgetMs - BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS),
+ interruptGraceMs:
+ options.interruptGraceMs ?? AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT,
+ flushReserveMs: BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,
+ reason: BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+ });
+ if (summary.tracked === 0) {
+ return;
+ }
+ if (summary.unsettled > 0) {
+ logger.warn('[background] Shutdown left background task results unconfirmed', summary);
+ return;
+ }
+ logger.info('[background] Drained background tasks for shutdown', summary);
+ },
+ /** After the generation job manager (100) finalizes interrupted turns, so completion
+ * deliveries anchored to them can resolve; before the subagent task store (90). */
+ { priority: 95 },
+ );
+}
+
/** Content for the synthetic ToolMessage returned when a call is backgrounded. */
export function buildBackgroundHandleContent(
task: Pick<BackgroundTask, 'id' | 'toolName' | 'status'>,
@@ -1830,6 +2020,9 @@ export function buildBackgroundCapacityContent(
message = `The server-wide background task registry is at capacity (running limit ${MAX_RUNNING_GLOBAL}). Retry later, or run this call in the foreground.`;
} else if (scope === 'global_retention') {
message = `The server-wide background task registry is retaining its maximum number of tasks (${MAX_TASKS_GLOBAL}), and pending result processing prevents safe eviction. Retry later, or run this call in the foreground.`;
+ } else if (scope === 'shutting_down') {
+ message =
+ 'This server is shutting down and is not starting new background tasks. Run this call in the foreground, or dispatch it again after the server restarts.';
} else if (scope === 'conversation_retention') {
message = `This conversation is retaining the maximum number of background tasks (${MAX_TASKS_PER_BUCKET}), and pending result processing prevents safe eviction. Wait for background result processing to finish, or run this call in the foreground.`;
} else {
diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts
index 8aabfefeab..8886ea89c0 100644
--- a/packages/api/src/agents/backgroundCompletion.ts
+++ b/packages/api/src/agents/backgroundCompletion.ts
@@ -6,6 +6,13 @@ export const BACKGROUND_TASK_ABORT_GRACE_MS: number = 60 * 1000;
/** Three missed heartbeats prove the process-local executor has been lost. */
export const BACKGROUND_TOOL_PRODUCER_LEASE_MS: number = 30_000;
export const BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS: number = 10_000;
+/** Recorded as the result of a background tool the server had to stop while shutting down. */
+export const BACKGROUND_TASK_SHUTDOWN_MESSAGE: string =
+ 'Background task was interrupted because its server shut down before it finished. Run it again if you still need the result.';
+/** Kept at the end of the background drain for the final durable receipt writes. */
+export const BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS: number = 2_000;
+/** Left after the background drain for the shutdown tasks that run after it. */
+export const BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS: number = 2_000;
/** Host-owned identity recorded before ordinary background tool work begins. */
export interface BackgroundToolWakeupRegistration {
diff --git a/packages/api/src/agents/handlers.shutdown.spec.ts b/packages/api/src/agents/handlers.shutdown.spec.ts
new file mode 100644
index 0000000000..fb9d6d1a00
--- /dev/null
+++ b/packages/api/src/agents/handlers.shutdown.spec.ts
@@ -0,0 +1,350 @@
+import { z } from 'zod';
+import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
+import type { BackgroundTaskRegistryClass } from './background';
+import type * as Handlers from './handlers';
+import { BACKGROUND_TASK_SHUTDOWN_MESSAGE } from './backgroundCompletion';
+
+type CreateToolExecuteHandler = typeof Handlers.createToolExecuteHandler;
+
+interface BatchResult {
+ content: string;
+}
+
+const flushMicrotasks = () => new Promise((resolve) => setImmediate(resolve));
+
+/** Each test drains the registry, which closes admission for the rest of the process,
+ * so every test loads its own handler and registry instances. */
+function loadModules(): {
+ createToolExecuteHandler: CreateToolExecuteHandler;
+ registry: BackgroundTaskRegistryClass;
+} {
+ let loaded:
+ | { createToolExecuteHandler: CreateToolExecuteHandler; registry: BackgroundTaskRegistryClass }
+ | undefined;
+ jest.isolateModules(() => {
+ const handlers = jest.requireActual<typeof Handlers>('./handlers');
+ const background = jest.requireActual<typeof import('./background')>('./background');
+ loaded = {
+ createToolExecuteHandler: handlers.createToolExecuteHandler,
+ registry: background.backgroundTaskRegistry,
+ };
+ });
+ if (loaded == null) {
+ throw new Error('modules failed to load');
+ }
+ return loaded;
+}
+
+const buildConfig = (backgroundToolNames: string[]) => ({
+ req: { user: { id: 'shutdown_user' }, body: { conversationId: 'shutdown_convo' } },
+ backgroundToolNames,
+});
+
+async function runBatch(
+ handler: ReturnType<CreateToolExecuteHandler>,
+ input: {
+ toolCalls: Array<Record<string, unknown>>;
+ configurable: Record<string, unknown>;
+ metadata: Record<string, unknown>;
+ },
+): Promise<BatchResult[]> {
+ let out: BatchResult[] = [];
+ await handler.handle('on_tool_execute', {
+ ...input,
+ agentId: 'agent_shutdown',
+ resolve: (results: BatchResult[]) => {
+ out = results;
+ },
+ reject: (error: Error) => {
+ throw error;
+ },
+ } as unknown as Parameters<typeof handler.handle>[1]);
+ return out;
+}
+
+function completionAdapter() {
+ const persistResult = jest.fn(async () => true);
+ const retire = jest.fn(async () => true);
+ const renew = jest.fn(async () => true);
+ const persist = jest.fn(async () => true);
+ return {
+ persistResult,
+ retire,
+ renew,
+ persist,
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({ renew, persistResult, retire })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
+ },
+ };
+}
+
+/** A search tool whose invocation the test controls; it may honor or ignore the abort. */
+function controlledTool(options: { honorAbort: boolean }) {
+ let resolveInvocation: (value: { content: string }) => void = () => undefined;
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(
+ (_input: unknown, config?: { signal?: AbortSignal }) =>
+ new Promise((resolve, reject) => {
+ resolveInvocation = resolve;
+ if (!options.honorAbort) {
+ return;
+ }
+ config?.signal?.addEventListener(
+ 'abort',
+ () => reject(new DOMException('This operation was aborted', 'AbortError')),
+ { once: true },
+ );
+ }),
+ ),
+ } as unknown as StructuredToolInterface;
+ return { tool, resolve: (content: string) => resolveInvocation({ content }) };
+}
+
+const drainOptions = () => ({
+ deadlineAt: Date.now() + 800,
+ interruptGraceMs: 300,
+ flushReserveMs: 100,
+ reason: BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+});
+
+const interruptedOutput = `Error: [search_mcp_docs] tool call failed: ${BACKGROUND_TASK_SHUTDOWN_MESSAGE}`;
+
+describe('createToolExecuteHandler — background tasks at shutdown', () => {
+ it('interrupts a running tool and delivers the shutdown reason as its result', async () => {
+ const { createToolExecuteHandler, registry } = loadModules();
+ const adapter = completionAdapter();
+ const { tool } = controlledTool({ honorAbort: true });
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: adapter.backgroundToolCompletion,
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-interrupt',
+ name: tool.name,
+ args: { q: 'slow', run_in_background: true },
+ stepId: 'step-interrupt',
+ },
+ ],
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-interrupt' },
+ });
+ const taskId = JSON.parse(dispatch.content).background_task_id;
+
+ const summary = await registry.drainForShutdown(drainOptions());
+
+ expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 0, unsettled: 0 });
+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);
+ expect(adapter.persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', output: interruptedOutput }),
+ );
+ expect(adapter.retire).not.toHaveBeenCalled();
+ expect(registry.get('shutdown_user', 'shutdown_convo', taskId)).toMatchObject({
+ status: 'error',
+ error: BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+ });
+ });
+
+ it('records an interrupted result for a tool that ignores the abort, and a late result keeps it', async () => {
+ const { createToolExecuteHandler, registry } = loadModules();
+ const adapter = completionAdapter();
+ const controlled = controlledTool({ honorAbort: false });
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [controlled.tool] }),
+ backgroundToolCompletion: adapter.backgroundToolCompletion,
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-stubborn',
+ name: controlled.tool.name,
+ args: { q: 'stubborn', run_in_background: true },
+ stepId: 'step-stubborn',
+ },
+ ],
+ configurable: buildConfig([controlled.tool.name]),
+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-stubborn' },
+ });
+ const taskId = JSON.parse(dispatch.content).background_task_id;
+
+ const summary = await registry.drainForShutdown(drainOptions());
+
+ expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 1, unsettled: 0 });
+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);
+ expect(adapter.persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'error', output: interruptedOutput }),
+ );
+ expect(registry.get('shutdown_user', 'shutdown_convo', taskId)?.status).toBe('error');
+
+ adapter.persist.mockResolvedValueOnce(false);
+ controlled.resolve('late result');
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);
+ expect(adapter.retire).not.toHaveBeenCalled();
+ });
+
+ it('stores a finished code result durably when its harvest is still waiting', async () => {
+ const { createToolExecuteHandler, registry } = loadModules();
+ const adapter = completionAdapter();
+ const codeTool = {
+ name: 'execute_code',
+ description: 'run code',
+ schema: z.object({ lang: z.string(), code: z.string() }),
+ invoke: jest.fn(async () => ({ content: 'stdout:\nhello' })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [codeTool] }),
+ backgroundToolCompletion: adapter.backgroundToolCompletion,
+ persistBackgroundCodeResult: () => new Promise(() => undefined),
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-code-harvest',
+ name: 'execute_code',
+ args: { lang: 'py', code: 'print(1)', run_in_background: true },
+ stepId: 'step-code-harvest',
+ turn: 1,
+ },
+ ],
+ configurable: buildConfig(['execute_code']),
+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-code-harvest' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+ expect(adapter.persistResult).not.toHaveBeenCalled();
+
+ const summary = await registry.drainForShutdown(drainOptions());
+
+ expect(summary).toEqual({ tracked: 1, interrupted: 0, flushed: 1, unsettled: 0 });
+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);
+ expect(adapter.persistResult).toHaveBeenCalledWith(
+ expect.objectContaining({ status: 'completed', output: expect.stringContaining('hello') }),
+ );
+ });
+
+ it('waits for the projected result when an independent receipt write returns false', async () => {
+ const { createToolExecuteHandler, registry } = loadModules();
+ const adapter = completionAdapter();
+ adapter.persistResult.mockResolvedValue(false);
+ let resolveProjection: (ready: boolean) => void = () => undefined;
+ adapter.persist.mockImplementation(
+ () =>
+ new Promise<boolean>((resolve) => {
+ resolveProjection = resolve;
+ }),
+ );
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(async () => ({ content: 'projected output' })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: adapter.backgroundToolCompletion,
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-projected',
+ name: tool.name,
+ args: { q: 'projected', run_in_background: true },
+ stepId: 'step-projected',
+ },
+ ],
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-projected' },
+ });
+ await flushMicrotasks();
+ expect(adapter.persistResult).toHaveReturnedTimes(1);
+ expect(adapter.persist).toHaveBeenCalledTimes(1);
+
+ const draining = registry.drainForShutdown(drainOptions());
+ resolveProjection(true);
+ expect(await draining).toEqual({ tracked: 1, interrupted: 0, flushed: 0, unsettled: 0 });
+ expect(adapter.retire).not.toHaveBeenCalled();
+ });
+
+ it('reports an unconfirmed result when neither receipt nor projection becomes durable', async () => {
+ const { createToolExecuteHandler, registry } = loadModules();
+ const adapter = completionAdapter();
+ adapter.persistResult.mockResolvedValue(false);
+ adapter.persist.mockResolvedValue(false);
+ const tool = {
+ name: 'search_mcp_docs',
+ description: 'search docs',
+ schema: z.object({ q: z.string() }),
+ invoke: jest.fn(async () => ({ content: 'not durable' })),
+ } as unknown as StructuredToolInterface;
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: adapter.backgroundToolCompletion,
+ });
+
+ await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-undurable',
+ name: tool.name,
+ args: { q: 'undurable', run_in_background: true },
+ stepId: 'step-undurable',
+ },
+ ],
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-undurable' },
+ });
+ await flushMicrotasks();
+ expect(adapter.retire).toHaveBeenCalledTimes(1);
+
+ expect(await registry.drainForShutdown(drainOptions())).toEqual({
+ tracked: 1,
+ interrupted: 0,
+ flushed: 1,
+ unsettled: 1,
+ });
+ });
+
+ it('refuses a new background dispatch once shutdown closes admission', async () => {
+ const { createToolExecuteHandler, registry } = loadModules();
+ const adapter = completionAdapter();
+ const { tool } = controlledTool({ honorAbort: true });
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: adapter.backgroundToolCompletion,
+ });
+ registry.closeAdmission();
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-after-shutdown',
+ name: tool.name,
+ args: { q: 'late', run_in_background: true },
+ stepId: 'step-after-shutdown',
+ },
+ ],
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-after-shutdown' },
+ });
+
+ expect(JSON.parse(dispatch.content)).toMatchObject({
+ status: 'rejected',
+ scope: 'shutting_down',
+ });
+ expect(tool.invoke).not.toHaveBeenCalled();
+ expect(adapter.backgroundToolCompletion.preregister).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index 969e886c1b..ab892079b7 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -96,6 +96,11 @@ import {
isCodeSessionToolName,
isFileResourceToolName,
} from './tools';
+import {
+ BACKGROUND_TASK_ABORT_GRACE_MS,
+ BACKGROUND_TASK_SHUTDOWN_MESSAGE,
+ BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
+} from './backgroundCompletion';
import {
createCodeApiRateLimitBudget,
isAbortError,
@@ -109,10 +114,6 @@ import {
contentFilterModelBoundBlockResponse,
isContentFilterError,
} from '~/middleware/contentFilter';
-import {
- BACKGROUND_TASK_ABORT_GRACE_MS,
- BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,
-} from './backgroundCompletion';
import {
WorkspaceToolHttpError,
WORKSPACE_EDIT_MAX_COUNT,
@@ -5888,7 +5889,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
};
}
const backgroundAbortController = new AbortController();
- let backgroundAbortSource: 'manual' | 'timeout' | undefined;
+ let backgroundAbortSource: 'manual' | 'timeout' | 'shutdown' | undefined;
const created = backgroundTaskRegistry.create({
...(detachedReservation?.status === 'reserved'
? { taskId: detachedReservation.taskId }
@@ -5959,6 +5960,62 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
);
}
}
+ let durableReceiptWrite: Promise<boolean> | undefined;
+ let durableReceiptAmbiguous = false;
+ let durableResultConfirmed = false;
+ let resolveDurableReceipt: () => void = () => undefined;
+ const durableReceiptSettled = new Promise<void>((resolve) => {
+ resolveDurableReceipt = resolve;
+ });
+ const confirmDurableResult = (): void => {
+ durableResultConfirmed = true;
+ resolveDurableReceipt();
+ };
+ /** One durable receipt per task. The task's own settlement and a
+ * shutdown flush share the first write, so neither can retire or
+ * contradict a receipt the other already stored. */
+ const writeDurableReceipt = (receipt: {
+ status: 'completed' | 'error' | 'cancelled';
+ output?: string;
+ settledAt: Date;
+ }): Promise<boolean> => {
+ if (durableReceiptWrite != null) {
+ return durableReceiptWrite;
+ }
+ durableReceiptWrite = (async (): Promise<boolean> => {
+ if (completionAdmission?.persistResult == null) {
+ return false;
+ }
+ try {
+ return await completionAdmission.persistResult({
+ status: receipt.status,
+ output: truncateMiddle(
+ receipt.output ?? '',
+ backgroundCompletionResultMaxChars,
+ ),
+ settledAt: receipt.settledAt,
+ });
+ } catch (receiptError) {
+ durableReceiptAmbiguous = true;
+ logger.warn(
+ `[background] Failed to persist independent result receipt for task ${task.id}:`,
+ receiptError,
+ );
+ return false;
+ }
+ })();
+ void durableReceiptWrite.then((written) => {
+ if (written) {
+ confirmDurableResult();
+ }
+ });
+ return durableReceiptWrite;
+ };
+ /** The task's own terminal result, recorded before its (possibly slow)
+ * persistence starts, so a shutdown flush can store it durably. */
+ let settledReceipt:
+ | { status: 'completed' | 'error' | 'cancelled'; output?: string }
+ | undefined;
/** Persists the settled result onto the dispatch turn's message
* (patch the tool-call part's output, persist generated files,
* append attachments), so a backgrounded code call reads like a
@@ -6021,37 +6078,18 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
);
const backgroundTask = resolveBackgroundTask();
let durableReceiptReady = false;
- let durableReceiptAmbiguous = false;
- const persistDurableReceipt = async (receipt: {
+ const persistDurableReceipt = (receipt: {
status: 'completed' | 'error' | 'cancelled';
output?: string;
- }): Promise<boolean> => {
- if (completionAdmission?.persistResult == null) {
- return false;
- }
- try {
- return await completionAdmission.persistResult({
- status: receipt.status,
- output: truncateMiddle(
- receipt.output ?? '',
- backgroundCompletionResultMaxChars,
- ),
- settledAt: backgroundTask.settledAt,
- });
- } catch (receiptError) {
- durableReceiptAmbiguous = true;
- logger.warn(
- `[background] Failed to persist independent result receipt for task ${task.id}:`,
- receiptError,
- );
- return false;
- }
- };
+ }): Promise<boolean> =>
+ writeDurableReceipt({ ...receipt, settledAt: backgroundTask.settledAt });
const retireFailedPersistence = async (
reason: string,
certainty: 'definite' | 'ambiguous',
): Promise<void> => {
- if (durableReceiptAmbiguous) {
+ /** Never retire a delivery that already holds a durable receipt,
+ * including one a shutdown flush stored ahead of this path. */
+ if (durableReceiptAmbiguous || (await durableReceiptWrite) === true) {
return;
}
if (completionAdmission == null) {
@@ -6136,6 +6174,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
'definite',
);
} else if (deliveryReady && !durableReceiptReady) {
+ confirmDurableResult();
completionAdmission?.expedite?.();
}
} catch (persistError) {
@@ -6207,6 +6246,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
output: params.output ?? localTask?.result,
});
}
+ if (persisted.deliveryReady !== false) {
+ confirmDurableResult();
+ }
if (persisted.deliveryReady === false) {
await retireFailedPersistence(
'background code result was not persisted',
@@ -6269,6 +6311,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
artifact?: unknown;
status: 'completed' | 'error' | 'cancelled';
}): Promise<void> => {
+ settledReceipt = { status: params.status, output: params.output };
/** Held for the whole persist, including a code harvest that waits for
* a long dispatch turn, so retention pressure cannot evict the task. */
backgroundTaskRegistry.markCompletionPersistencePending(
@@ -6431,7 +6474,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}, BACKGROUND_TASK_ABORT_GRACE_MS);
producerRetirementTimeout.unref?.();
};
- void (async () => {
+ const settlement = (async () => {
try {
const result = await withBackgroundTaskTimeout(
invokePromise,
@@ -6552,7 +6595,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
backgroundControlEnabled,
),
);
- const errorOutput = policyError ?? message;
+ /** Tools report an abort in their own words; a shutdown interrupt
+ * tells the agent why it happened and that it may retry. */
+ const errorOutput =
+ backgroundAbortSource === 'shutdown'
+ ? BACKGROUND_TASK_SHUTDOWN_MESSAGE
+ : (policyError ?? message);
const filteredError =
policyError == null
? filteredToolOutputResult(tc, backgroundReq, {
@@ -6621,6 +6669,64 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
await stopProducerHeartbeat();
}
})();
+ void settlement.then(
+ () => {
+ if (completionAdmission?.persistResult == null || !completionPreregistered) {
+ resolveDurableReceipt();
+ }
+ },
+ () => undefined,
+ );
+ backgroundTaskRegistry.trackShutdown(task, {
+ settled: durableReceiptSettled,
+ interrupt: (reason) => {
+ if (backgroundAbortSource != null || backgroundAbortController.signal.aborted) {
+ return;
+ }
+ backgroundAbortSource = 'shutdown';
+ backgroundAbortController.abort(new DOMException(reason, 'AbortError'));
+ },
+ flush: async (reason) => {
+ await stopProducerHeartbeat();
+ if (durableReceiptWrite != null) {
+ await durableReceiptWrite;
+ } else if (settledReceipt != null) {
+ await writeDurableReceipt({ ...settledReceipt, settledAt: new Date() });
+ } else {
+ const failure = toBackgroundToolFailure(tc.name, reason);
+ backgroundTaskRegistry.fail(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ isCodeCall ? failure : reason,
+ { harvestStarted: harvestEnabled },
+ );
+ try {
+ await persistDetachedTerminal({ status: 'failed', error: failure });
+ } catch (detachedError) {
+ logger.warn(
+ `[background] Failed to settle detached action for interrupted task ${task.id}:`,
+ detachedError,
+ );
+ }
+ await writeDurableReceipt({
+ status: 'error',
+ output: failure,
+ settledAt: new Date(),
+ });
+ }
+ if (durableResultConfirmed || completionAdmission?.persistResult == null) {
+ return;
+ }
+ if (settledReceipt != null) {
+ await settlement;
+ if (durableResultConfirmed) {
+ return;
+ }
+ }
+ throw new Error(`Background task ${task.id} has no durable shutdown result`);
+ },
+ });
if (
detachedReservation?.status === 'reserved' &&
eventActorDetachedAction != null &&
diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.ts
index 6b0cfc3a3c..5869bdf582 100644
--- a/packages/api/src/agents/index.ts
+++ b/packages/api/src/agents/index.ts
@@ -83,7 +83,7 @@ export * from './activityPhases';
export * from './subagentDelivery';
export * from './view';
export * from './tasks';
-export { backgroundTaskRegistry } from './background';
+export { backgroundTaskRegistry, registerBackgroundTaskShutdown } from './background';
export * from './workspace';
export * from './reasoningLabels';
export * from './refusal';
diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.ts
index 43486a54b6..c62b09e8ce 100644
--- a/packages/api/src/agents/triggers/service.delivery.spec.ts
+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts
@@ -292,6 +292,39 @@ describe('durable agent trigger service', () => {
await service.stop();
});
+ it('keeps background producer writes available after the delivery engine stops for shutdown', async () => {
+ const methods = deliveryMethods({
+ persistAgentBackgroundToolResult: jest.fn(async () => true),
+ renewAgentTriggerDeliveryProducerLease: jest.fn(async () => true),
+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),
+ });
+ const service = createAgentTriggerService({ methods });
+ await service.initialize({ address });
+ await service.stop();
+
+ await expect(
+ service.renewProducerLease(
+ 'trigger_background',
+ 'background-tool-completion',
+ new Date(START.getTime() + 30_000),
+ ),
+ ).resolves.toBe(true);
+ await expect(
+ service.persistBackgroundToolResult({
+ deliveryKey: 'trigger_background',
+ sourceId: 'background-tool-completion',
+ result,
+ }),
+ ).resolves.toBe(true);
+ await flush();
+
+ expect(methods.persistAgentBackgroundToolResult).toHaveBeenCalledTimes(1);
+ expect(methods.expediteAgentTriggerDeliveries).not.toHaveBeenCalled();
+ await expect(service.enqueue(envelope())).rejects.toBeInstanceOf(
+ AgentTriggerServiceUnavailableError,
+ );
+ });
+
it("expedites a principal's completion deliveries when one of its generations settles", async () => {
const methods = deliveryMethods({
expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })),
diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.ts
index 53de18fc35..f2786ed860 100644
--- a/packages/api/src/agents/triggers/service.ts
+++ b/packages/api/src/agents/triggers/service.ts
@@ -783,9 +783,12 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
}
return retired;
}),
+ // A background tool keeps running while the server drains, and its result must
+ // still reach the durable receipt, so producer writes stay available after
+ // admissions and the delivery engine stop.
renewProducerLease: (deliveryKey, sourceId, leaseUntil) =>
runAsSystem(async () =>
- requireMethods().renewAgentTriggerDeliveryProducerLease({
+ requireCleanupMethods().renewAgentTriggerDeliveryProducerLease({
deliveryKey,
sourceId,
leaseUntil,
@@ -793,7 +796,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
),
persistBackgroundToolResult: (input) =>
runAsSystem(async () => {
- const persist = requireMethods().persistAgentBackgroundToolResult;
+ const persist = requireCleanupMethods().persistAgentBackgroundToolResult;
const persisted = persist == null ? false : await persist(input);
if (persisted) {
expediteCompletions({ deliveryKeys: [input.deliveryKey] });
diff --git a/packages/api/src/app/shutdown.spec.ts b/packages/api/src/app/shutdown.spec.ts
index 3b1a4631e9..7ede31b3bc 100644
--- a/packages/api/src/app/shutdown.spec.ts
+++ b/packages/api/src/app/shutdown.spec.ts
@@ -4,6 +4,7 @@ import {
setupGracefulShutdown,
isShutdownInProgress,
registerShutdownTask,
+ getClusterShutdownBudgetMs,
__resetShutdownStateForTests,
} from './shutdown';
@@ -18,6 +19,52 @@ const triggerSignal = (signal: NodeJS.Signals): void => {
const flush = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));
+describe('getClusterShutdownBudgetMs', () => {
+ it('prefers the primary force-exit deadline over the worker timer', () => {
+ expect(
+ getClusterShutdownBudgetMs({
+ deadlineAt: 14_000,
+ forceExitMs: 30_000,
+ remainingMs: 60_000,
+ elapsedMs: 1_000,
+ now: 10_000,
+ }),
+ ).toBe(4_000);
+ });
+
+ it('uses the elapsed estimate when the primary deadline was not received', () => {
+ expect(
+ getClusterShutdownBudgetMs({
+ deadlineAt: null,
+ forceExitMs: 30_000,
+ remainingMs: 60_000,
+ elapsedMs: 2_000,
+ }),
+ ).toBe(28_000);
+ });
+
+ it('honors the tighter local limit and requires an active shutdown', () => {
+ expect(
+ getClusterShutdownBudgetMs({
+ deadlineAt: 15_000,
+ forceExitMs: 30_000,
+ remainingMs: 1_000,
+ elapsedMs: 5_000,
+ now: 10_000,
+ }),
+ ).toBe(1_000);
+ expect(
+ getClusterShutdownBudgetMs({
+ deadlineAt: 15_000,
+ forceExitMs: 30_000,
+ remainingMs: null,
+ elapsedMs: 5_000,
+ now: 10_000,
+ }),
+ ).toBeNull();
+ });
+});
+
describe('setupGracefulShutdown', () => {
let server: http.Server;
let exitSpy: jest.SpyInstance;
diff --git a/packages/api/src/app/shutdown.ts b/packages/api/src/app/shutdown.ts
index 63216a1053..7c56981565 100644
--- a/packages/api/src/app/shutdown.ts
+++ b/packages/api/src/app/shutdown.ts
@@ -68,6 +68,25 @@ export function getRemainingShutdownMs(): number | null {
return Math.max(0, SHUTDOWN_TIMEOUT_MS - (Date.now() - shutdownStartedAt));
}
+export function getClusterShutdownBudgetMs({
+ deadlineAt,
+ forceExitMs,
+ remainingMs = getRemainingShutdownMs(),
+ elapsedMs = getShutdownElapsedMs(),
+ now = Date.now(),
+}: {
+ deadlineAt: number | null;
+ forceExitMs: number;
+ remainingMs?: number | null;
+ elapsedMs?: number | null;
+ now?: number;
+}): number | null {
+ if (remainingMs == null || elapsedMs == null) {
+ return null;
+ }
+ return Math.min(remainingMs, deadlineAt == null ? forceExitMs - elapsedMs : deadlineAt - now);
+}
+
export function isShutdownInProgress(): boolean {
return isShuttingDown;
}
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index 7ff73ec7a8..5e0eead7ce 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -771,6 +771,7 @@ describe('agent background task config', () => {
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
+ shutdownInterruptGraceMs: 5_000,
});
});
@@ -789,6 +790,7 @@ describe('agent background task config', () => {
completionWakeups: false,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
+ shutdownInterruptGraceMs: 5_000,
});
});
@@ -807,9 +809,31 @@ describe('agent background task config', () => {
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: true,
+ shutdownInterruptGraceMs: 5_000,
});
});
+ it('accepts a bounded shutdown interrupt grace', () => {
+ const result = configSchema.safeParse({
+ version: '1.0',
+ endpoints: { agents: { backgroundTasks: { shutdownInterruptGraceMs: 0 } } },
+ });
+
+ expect(result.success).toBe(true);
+ if (result.success) {
+ expect(result.data.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs).toBe(0);
+ }
+ });
+
+ it.each([-1, 60_001, 1.5])('rejects an unsafe shutdown interrupt grace: %s', (graceMs) => {
+ expect(
+ configSchema.safeParse({
+ version: '1.0',
+ endpoints: { agents: { backgroundTasks: { shutdownInterruptGraceMs: graceMs } } },
+ }).success,
+ ).toBe(false);
+ });
+
it('accepts a bounded durable completion result limit', () => {
const result = configSchema.safeParse({
version: '1.0',
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index d5b8eb0185..7afb9cb280 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -25,6 +25,7 @@ import {
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
+export const AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT = 5_000;
import {
MAX_SUBAGENTS,
MAX_SUBAGENTS_CEILING,
@@ -1575,6 +1576,15 @@ export const agentsEndpointSchema = baseEndpointSchema
/** Cooperative cancellation for process-local ordinary tools. Off
* by default so existing deployments opt into the new control. */
ordinaryToolCancellation: z.boolean().optional().default(false),
+ /** During graceful shutdown, how long an interrupted background tool gets to
+ * settle on its own before its result is recorded as interrupted. */
+ shutdownInterruptGraceMs: z
+ .number()
+ .int()
+ .min(0)
+ .max(60_000)
+ .optional()
+ .default(AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT),
})
.optional(),
skills: z