Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🛬 fix: Settle Background Tool Results Before the Server Shuts Down (#16365)
Danny Avilac606e5065a65
diff --git a/api/server/experimental.js b/api/server/experimental.jsindex 45332738d4..276beb8ed0 100644--- a/api/server/experimental.js+++ b/api/server/experimental.js@@ -43,8 +43,8 @@ const {configureServerTimeouts,setupGracefulShutdown,registerShutdownTask,- getRemainingShutdownMs,- getShutdownElapsedMs,+ getClusterShutdownBudgetMs,+ registerBackgroundTaskShutdown,configureMessageFilterRegexValidator,configureFileConfigRegexEngine,configureAgentEventRuntime,@@ -386,24 +386,18 @@ if (cluster.isMaster) {* after signalling. Measure against that, and hold back a reserve for the tasks after this* one. Abandoning an unrecorded drain fences the next generation permanently. */const CLUSTER_TEARDOWN_RESERVE_MS = 3_000;+ const clusterShutdownBudgetMs = () =>+ getClusterShutdownBudgetMs({+ deadlineAt: clusterShutdownDeadlineAt,+ forceExitMs: CLUSTER_FORCE_EXIT_MS,+ });const destroyGenerationJobManager = () => {- const remaining = getRemainingShutdownMs();- const elapsed = getShutdownElapsedMs();- if (remaining == null || elapsed == null) {+ const budgetMs = clusterShutdownBudgetMs();+ if (budgetMs == null) {return GenerationJobManager.destroy();}- /** Prefer the deadline the primary actually set. The elapsed-based estimate starts- * counting only when this worker's signal handler ran, which lags the primary's timer- * by however long the event loop was blocked. */- const primaryRemaining =- clusterShutdownDeadlineAt != null- ? clusterShutdownDeadlineAt - Date.now()- : CLUSTER_FORCE_EXIT_MS - elapsed;return GenerationJobManager.destroy({- settlementBudgetMs: Math.max(- 0,- Math.min(remaining, primaryRemaining) - CLUSTER_TEARDOWN_RESERVE_MS,- ),+ settlementBudgetMs: Math.max(0, budgetMs - CLUSTER_TEARDOWN_RESERVE_MS),});};// Tear down stream resources before shared caches and telemetry exporters shut down.@@ -508,6 +502,10 @@ if (cluster.isMaster) {// principal) still merges DB `__base__` overrides, which must not drive which hook// modules load in every worker (matches api/server/index.js's baseOnly usage).const baseAppConfig = await getAppConfig({ baseOnly: true });+ registerBackgroundTaskShutdown({+ interruptGraceMs: baseAppConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs,+ getBudgetMs: clusterShutdownBudgetMs,+ });configureAgentEventRuntime(baseAppConfig?.endpoints?.agents?.eventDriven);const toolApproval = baseAppConfig?.endpoints?.agents?.toolApproval;await loadToolApprovalHooks(toolApproval?.enabled ? toolApproval.hooks : undefined, {diff --git a/api/server/experimental.spec.js b/api/server/experimental.spec.jsindex b1f05d4663..30d2878a09 100644--- a/api/server/experimental.spec.js+++ b/api/server/experimental.spec.js@@ -23,6 +23,18 @@ describe('Experimental server configuration', () => {expect(source).toMatch(/if \(shuttingDown\) \{[\s\S]*?return;[\s\S]*?Starting a new worker/);});+ it("drains background tasks within the primary's cluster shutdown deadline", () => {+ expect(source).toMatch(+ /registerBackgroundTaskShutdown\(\{[\s\S]*?getBudgetMs: clusterShutdownBudgetMs,[\s\S]*?\}\);/,+ );+ expect(source).toMatch(+ /const clusterShutdownBudgetMs = \(\) =>\s*getClusterShutdownBudgetMs\(\{\s*deadlineAt: clusterShutdownDeadlineAt,\s*forceExitMs: CLUSTER_FORCE_EXIT_MS,\s*\}\);/,+ );+ expect(source).toMatch(+ /const destroyGenerationJobManager = \(\) => \{\s*const budgetMs = clusterShutdownBudgetMs\(\);/,+ );+ });+it('starts approval expiry after installing the scheduled-run callback', () => {const handlerIndex = source.indexOf('GenerationJobManager.setApprovalExpiredHandler(recordExpiredScheduleApproval);',diff --git a/api/server/index.js b/api/server/index.jsindex 1d31f63b26..fa3412b0f7 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -48,6 +48,7 @@ const {requestContextMiddleware,registerShutdownTask,getRemainingShutdownMs,+ registerBackgroundTaskShutdown,configureServerTimeouts,setupGracefulShutdown,updateInterfacePermissions,@@ -231,6 +232,9 @@ const startServer = async () => {logger.error('[sweepOrphanedPreviews] Background sweep failed:', err);});const appConfig = await getAppConfig({ baseOnly: true });+ registerBackgroundTaskShutdown({+ interruptGraceMs: appConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs,+ });configureAgentEventRuntime(appConfig?.endpoints?.agents?.eventDriven);warnOnUnreachableDeliveryPaths(appConfig);initializeFileStorage(appConfig);diff --git a/api/server/index.spec.js b/api/server/index.spec.jsindex e1b565dcc3..5f5a54c475 100644--- a/api/server/index.spec.js+++ b/api/server/index.spec.js@@ -188,6 +188,17 @@ describe('Startup readiness wiring', () => {expect(shutdownRegistrationIndex).toBeLessThan(listenIndex);});+ it('registers background task draining with the graceful shutdown coordinator', () => {+ const registrationIndex = source.indexOf('registerBackgroundTaskShutdown({');+ const listenIndex = source.indexOf('const server = app.listen');++ expect(registrationIndex).toBeGreaterThan(-1);+ expect(registrationIndex).toBeLessThan(listenIndex);+ expect(source).toContain(+ 'interruptGraceMs: appConfig?.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs',+ );+ });+it('configures HTTP timeouts before graceful shutdown handling', () => {const listenIndex = source.indexOf('const server = app.listen');const timeoutConfigIndex = source.indexOf('configureServerTimeouts(server);');diff --git a/packages/api/src/agents/background.shutdown.spec.ts b/packages/api/src/agents/background.shutdown.spec.tsnew file mode 100644index 0000000000..83d1684b02--- /dev/null+++ b/packages/api/src/agents/background.shutdown.spec.ts@@ -0,0 +1,281 @@+import { AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT } from 'librechat-data-provider';+import type { BackgroundTask, BackgroundTaskShutdownHandle } from './background';+import {+ BACKGROUND_TASK_SHUTDOWN_MESSAGE,+ BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,+ BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS,+} from './backgroundCompletion';+import {+ BackgroundTaskRegistryClass,+ buildBackgroundCapacityContent,+ registerBackgroundTaskShutdown,+} from './background';+import * as shutdown from '~/app/shutdown';++const REASON = 'server shutting down';++function createTask(registry: BackgroundTaskRegistryClass, toolCallId: string): BackgroundTask {+ const created = registry.create({+ userId: 'shutdown-owner',+ conversationId: 'shutdown-conversation',+ toolCallId,+ toolName: 'bash_tool',+ });+ if ('atCapacity' in created) {+ throw new Error('unexpected capacity');+ }+ return created.task;+}++interface ControlledHandle {+ handle: BackgroundTaskShutdownHandle;+ interrupt: jest.Mock;+ flush: jest.Mock;+ settle: () => void;+}++/** A handle whose settlement the test drives; `onInterrupt`/`flushResult` model the tool. */+function controlledHandle(+ options: { onInterrupt?: 'settle' | 'ignore'; flushResult?: 'settle' | 'hang' } = {},+): ControlledHandle {+ let settle: () => void = () => undefined;+ const settled = new Promise<void>((resolve) => {+ settle = resolve;+ });+ const interrupt = jest.fn(() => {+ if (options.onInterrupt === 'settle') {+ settle();+ }+ });+ const flush = jest.fn(() => {+ if (options.flushResult === 'hang') {+ return new Promise<void>(() => undefined);+ }+ settle();+ return Promise.resolve();+ });+ return { handle: { settled, interrupt, flush }, interrupt, flush, settle: () => settle() };+}++const drainOptions = (overrides: { deadlineMs?: number; interruptGraceMs?: number } = {}) => ({+ deadlineAt: Date.now() + (overrides.deadlineMs ?? 400),+ interruptGraceMs: overrides.interruptGraceMs ?? 100,+ flushReserveMs: 50,+ reason: REASON,+});++describe('background task shutdown', () => {+ describe('admission', () => {+ it('refuses new tasks once admission closes, but still resolves replays and reserved permits', () => {+ const registry = new BackgroundTaskRegistryClass();+ const existing = createTask(registry, 'before-shutdown');+ const permitAdmission = registry.reserveCapacity({+ userId: 'shutdown-owner',+ conversationId: 'shutdown-conversation',+ toolCallId: 'reserved-before-shutdown',+ });+ if (!('permit' in permitAdmission)) {+ throw new Error('expected a capacity permit');+ }++ registry.closeAdmission();++ expect(registry.isAdmissionClosed()).toBe(true);+ expect(+ registry.create({+ userId: 'shutdown-owner',+ conversationId: 'shutdown-conversation',+ toolCallId: 'after-shutdown',+ toolName: 'bash_tool',+ }),+ ).toEqual({ atCapacity: true, scope: 'shutting_down' });+ expect(+ registry.reserveCapacity({+ userId: 'shutdown-owner',+ conversationId: 'shutdown-conversation',+ toolCallId: 'after-shutdown',+ }),+ ).toEqual({ atCapacity: true, scope: 'shutting_down' });+ expect(+ registry.create({+ userId: 'shutdown-owner',+ conversationId: 'shutdown-conversation',+ toolCallId: 'before-shutdown',+ toolName: 'bash_tool',+ }),+ ).toEqual({ task: existing, isNew: false });+ expect(+ registry.create({+ userId: 'shutdown-owner',+ conversationId: 'shutdown-conversation',+ toolCallId: 'reserved-before-shutdown',+ toolName: 'bash_tool',+ capacityPermit: permitAdmission.permit,+ }),+ ).toMatchObject({ isNew: true });+ });++ it('tells the model why a background dispatch was refused', () => {+ const content = JSON.parse(buildBackgroundCapacityContent('bash_tool', 'shutting_down'));+ expect(content).toEqual({+ status: 'rejected',+ tool: 'bash_tool',+ scope: 'shutting_down',+ message: expect.stringContaining('shutting down'),+ });+ });+ });++ describe('drainForShutdown', () => {+ it('returns immediately and closes admission when nothing is tracked', async () => {+ const registry = new BackgroundTaskRegistryClass();+ const startedAt = Date.now();++ const summary = await registry.drainForShutdown(drainOptions({ deadlineMs: 5_000 }));++ expect(summary).toEqual({ tracked: 0, interrupted: 0, flushed: 0, unsettled: 0 });+ expect(Date.now() - startedAt).toBeLessThan(1_000);+ expect(registry.isAdmissionClosed()).toBe(true);+ });++ it('waits for tasks that settle on their own without interrupting them', async () => {+ const registry = new BackgroundTaskRegistryClass();+ const task = createTask(registry, 'settles-alone');+ const controlled = controlledHandle();+ registry.trackShutdown(task, controlled.handle);+ setTimeout(controlled.settle, 20);++ const summary = await registry.drainForShutdown(drainOptions());++ expect(summary).toEqual({ tracked: 1, interrupted: 0, flushed: 0, unsettled: 0 });+ expect(controlled.interrupt).not.toHaveBeenCalled();+ expect(controlled.flush).not.toHaveBeenCalled();+ });++ it('interrupts running tasks, then flushes only those that did not settle after the abort', async () => {+ const registry = new BackgroundTaskRegistryClass();+ const cooperative = controlledHandle({ onInterrupt: 'settle' });+ const stubborn = controlledHandle({ onInterrupt: 'ignore' });+ registry.trackShutdown(createTask(registry, 'cooperative'), cooperative.handle);+ registry.trackShutdown(createTask(registry, 'stubborn'), stubborn.handle);++ const summary = await registry.drainForShutdown(drainOptions());++ expect(cooperative.interrupt).toHaveBeenCalledWith(REASON);+ expect(stubborn.interrupt).toHaveBeenCalledWith(REASON);+ expect(cooperative.flush).not.toHaveBeenCalled();+ expect(stubborn.flush).toHaveBeenCalledWith(REASON);+ expect(summary).toEqual({ tracked: 2, interrupted: 2, flushed: 1, unsettled: 0 });+ });++ it('flushes a settled task whose result is not durable yet without aborting it', async () => {+ const registry = new BackgroundTaskRegistryClass();+ const task = createTask(registry, 'settled-not-durable');+ const controlled = controlledHandle();+ registry.trackShutdown(task, controlled.handle);+ registry.complete('shutdown-owner', 'shutdown-conversation', task.id, { content: 'done' });++ const summary = await registry.drainForShutdown(drainOptions());++ expect(controlled.interrupt).not.toHaveBeenCalled();+ expect(controlled.flush).toHaveBeenCalledWith(REASON);+ expect(summary).toEqual({ tracked: 1, interrupted: 0, flushed: 1, unsettled: 0 });+ });++ it('stops at the deadline and reports results it could not confirm', async () => {+ const registry = new BackgroundTaskRegistryClass();+ const hanging = controlledHandle({ flushResult: 'hang' });+ registry.trackShutdown(createTask(registry, 'hanging'), hanging.handle);+ const startedAt = Date.now();++ const summary = await registry.drainForShutdown(drainOptions({ deadlineMs: 300 }));++ expect(Date.now() - startedAt).toBeLessThan(2_000);+ expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 1, unsettled: 1 });+ });++ it('stops tracking a task once its result is durable', async () => {+ const registry = new BackgroundTaskRegistryClass();+ const controlled = controlledHandle();+ registry.trackShutdown(createTask(registry, 'released'), controlled.handle);++ controlled.settle();+ await controlled.handle.settled;+ await Promise.resolve();++ const summary = await registry.drainForShutdown(drainOptions());+ expect(summary.tracked).toBe(0);+ });+ });++ describe('registerBackgroundTaskShutdown', () => {+ type RegisteredTask = [string, () => void | Promise<void>, shutdown.ShutdownTaskOptions?];++ function captureRegistrations(): RegisteredTask[] {+ const registered: RegisteredTask[] = [];+ jest.spyOn(shutdown, 'registerShutdownTask').mockImplementation((name, fn, options) => {+ registered.push([name, fn, options]);+ });+ return registered;+ }++ afterEach(() => {+ jest.restoreAllMocks();+ });++ it('closes admission before the HTTP drain and settles tasks after generations finalize', async () => {+ const registered = captureRegistrations();+ const registry = new BackgroundTaskRegistryClass();+ const drain = jest.spyOn(registry, 'drainForShutdown');++ registerBackgroundTaskShutdown({ registry, getBudgetMs: () => 10_000 });++ expect(registered.map(([name, , options]) => [name, options])).toEqual([+ ['background task admission', { phase: 'pre-drain', priority: 110 }],+ ['background tasks', { priority: 95 }],+ ]);+ expect(registered[0][2]?.priority).toBeGreaterThan(100);+ registered[0][1]();+ expect(registry.isAdmissionClosed()).toBe(true);++ const startedAt = Date.now();+ await registered[1][1]();+ expect(drain).toHaveBeenCalledWith({+ deadlineAt: expect.any(Number),+ interruptGraceMs: AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT,+ flushReserveMs: BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,+ reason: BACKGROUND_TASK_SHUTDOWN_MESSAGE,+ });+ const { deadlineAt } = drain.mock.calls[0][0];+ const expected = startedAt + 10_000 - BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS;+ expect(Math.abs(deadlineAt - expected)).toBeLessThan(1_000);+ });++ it('uses the configured interrupt grace', async () => {+ const registered = captureRegistrations();+ const registry = new BackgroundTaskRegistryClass();+ const drain = jest.spyOn(registry, 'drainForShutdown');++ registerBackgroundTaskShutdown({+ registry,+ interruptGraceMs: 1_234,+ getBudgetMs: () => 5_000,+ });+ await registered[1][1]();++ expect(drain.mock.calls[0][0].interruptGraceMs).toBe(1_234);+ });++ it('only closes admission when no shutdown budget is known', async () => {+ const registered = captureRegistrations();+ const registry = new BackgroundTaskRegistryClass();+ const drain = jest.spyOn(registry, 'drainForShutdown');++ registerBackgroundTaskShutdown({ registry, getBudgetMs: () => null });+ await registered[1][1]();++ expect(drain).not.toHaveBeenCalled();+ expect(registry.isAdmissionClosed()).toBe(true);+ });+ });+});diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.tsindex c95cb89524..700a33e730 100644--- a/packages/api/src/agents/background.ts+++ b/packages/api/src/agents/background.ts@@ -36,7 +36,12 @@import { logger } from '@librechat/data-schemas';import { createHash, randomUUID } from 'node:crypto';import { Constants as AgentConstants } from '@librechat/agents';-import { Tools, Constants, imageGenTools } from 'librechat-data-provider';+import {+ Tools,+ Constants,+ imageGenTools,+ AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT,+} from 'librechat-data-provider';import type {LCTool,LCToolRegistry,@@ -57,6 +62,9 @@ import type { BackgroundToolResultState } from './harvest';import type { CapabilityToolNames } from './selection';import {BACKGROUND_TASK_TIMEOUT_MS,+ BACKGROUND_TASK_SHUTDOWN_MESSAGE,+ BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,+ BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS,type PendingBackgroundCompletion,type BackgroundToolDeadClaimRecovery,type BackgroundToolWakeupAdmission,@@ -75,6 +83,7 @@ import {synthesizeSelectionToolOptions,} from './selection';import { SUBAGENT_WAKEUP_GUIDANCE, agentUsesSubagentCompletionWakeups } from './subagentDelivery';+import { registerShutdownTask, getRemainingShutdownMs } from '~/app/shutdown';import { SubagentTaskOwnerUnavailableError } from './subagentTaskRouting';import { SET_MEMORY_TOOL_NAME, DELETE_MEMORY_TOOL_NAME } from './memory';import { ASK_USER_QUESTION_TOOL_NAME } from './hitl/askUserQuestionTool';@@ -711,13 +720,53 @@ export type BackgroundTaskCapacityScope =| 'user_running'| 'user_retention'| 'global_running'- | 'global_retention';+ | 'global_retention'+ | 'shutting_down';type BackgroundTaskCapacityRejection = {atCapacity: true;scope: BackgroundTaskCapacityScope;};+/** Process-local controls a running task registers so graceful shutdown can settle it. */+export interface BackgroundTaskShutdownHandle {+ /** Resolves once the task's result is durable, or once the task finished when it has+ * no durable delivery. Never rejects. */+ settled: Promise<void>;+ /** Aborts the live invocation because the server is shutting down. */+ interrupt: (reason: string) => void;+ /** Stores the best result available now: the task's own result when it has settled,+ * otherwise an interrupted failure. */+ flush: (reason: string) => Promise<void>;+}++export interface BackgroundTaskDrainSummary {+ /** Tasks with an unsettled result when the drain began. */+ tracked: number;+ /** Tasks still running when the drain aborted them. */+ interrupted: number;+ /** Tasks whose result was still not durable, so the drain stored one. */+ flushed: number;+ /** Tasks whose durable result was still unconfirmed at the deadline. */+ unsettled: number;+}++/** Resolves when `promise` settles or at `deadlineAt`, whichever comes first. */+function waitUntil(promise: Promise<unknown>, deadlineAt: number): Promise<void> {+ const remainingMs = deadlineAt - Date.now();+ if (remainingMs <= 0) {+ return Promise.resolve();+ }+ return new Promise<void>((resolve) => {+ const timer = setTimeout(resolve, remainingMs);+ const finish = (): void => {+ clearTimeout(timer);+ resolve();+ };+ promise.then(finish, finish);+ });+}+const COMPLETED_TASK_TTL_MS = 60 * 60 * 1000;const IDLE_BUCKET_TTL_MS = 6 * 60 * 60 * 1000;const MAX_RUNNING_PER_BUCKET = 10;@@ -829,6 +878,9 @@ export class BackgroundTaskRegistryClass {/** Live invocation controls are intentionally process-local and are never* exposed through task snapshots or durable receipts. */private readonly cancellationRequests = new WeakMap<BackgroundTask, () => boolean | void>();+ /** Tasks whose result is not yet durable, for the graceful-shutdown drain. */+ private readonly shutdownHandles = new Map<BackgroundTask, BackgroundTaskShutdownHandle>();+ private admissionClosed = false;private lastGlobalSweepAt = 0;private key(userId: string, conversationId: string): string {@@ -1171,6 +1223,9 @@ export class BackgroundTaskRegistryClass {if (existing != null) {return { task: existing, isNew: false };}+ if (this.admissionClosed) {+ return { atCapacity: true, scope: 'shutting_down' };+ }if (existingBucket != null &&this.runningCount(existingBucket) + existingBucket.capacityPermits.size >=@@ -1261,6 +1316,11 @@ export class BackgroundTaskRegistryClass {}}+ /** A permit already reserved launch authority before shutdown began, so it may still land. */+ if (this.admissionClosed && params.capacityPermit == null) {+ return { atCapacity: true, scope: 'shutting_down' };+ }+if (params.capacityPermit != null) {if (existingBucket == null) {throw new Error('Background task capacity permit is stale');@@ -1348,6 +1408,85 @@ export class BackgroundTaskRegistryClass {return { status: 'requested', task };}+ /** Refuses new background tasks; the server is shutting down. Replays still resolve. */+ closeAdmission(): void {+ this.admissionClosed = true;+ }++ isAdmissionClosed(): boolean {+ return this.admissionClosed;+ }++ /** Registers a new task's shutdown controls until its result is durable. */+ trackShutdown(task: BackgroundTask, handle: BackgroundTaskShutdownHandle): void {+ this.shutdownHandles.set(task, handle);+ const release = (): void => {+ if (this.shutdownHandles.get(task) === handle) {+ this.shutdownHandles.delete(task);+ }+ };+ handle.settled.then(release, release);+ }++ /**+ * Settles every tracked task before the process exits. Tasks first get whatever time+ * is left before `interruptGraceMs` and the flush reserve; tasks still running are then+ * aborted and get `interruptGraceMs` to settle on their own; the rest have a durable+ * result written for them. Without this, a restart abandons them and their automatic+ * deliveries dead-letter once the producer lease expires.+ */+ async drainForShutdown(options: {+ deadlineAt: number;+ interruptGraceMs: number;+ flushReserveMs: number;+ reason: string;+ }): Promise<BackgroundTaskDrainSummary> {+ this.closeAdmission();+ const tracked = this.shutdownHandles.size;+ if (tracked === 0) {+ return { tracked: 0, interrupted: 0, flushed: 0, unsettled: 0 };+ }+ const flushAt = options.deadlineAt - options.flushReserveMs;+ const interruptAt = flushAt - options.interruptGraceMs;++ await this.waitForShutdownHandles(interruptAt);+ let interrupted = 0;+ for (const [task, handle] of this.shutdownHandles) {+ if (task.status === 'running') {+ interrupted++;+ handle.interrupt(options.reason);+ }+ }++ await this.waitForShutdownHandles(flushAt);+ const remaining = [...this.shutdownHandles.values()];+ let durable = 0;+ await waitUntil(+ Promise.allSettled(+ remaining.map((handle) =>+ handle.flush(options.reason).then(() => {+ durable++;+ }),+ ),+ ),+ options.deadlineAt,+ );+ return {+ tracked,+ interrupted,+ flushed: remaining.length,+ unsettled: remaining.length - durable,+ };+ }++ private waitForShutdownHandles(deadlineAt: number): Promise<void> {+ const pending = [...this.shutdownHandles.values()].map((handle) => handle.settled);+ if (pending.length === 0) {+ return Promise.resolve();+ }+ return waitUntil(Promise.all(pending), deadlineAt);+ }+private update(userId: string,conversationId: string,@@ -1795,6 +1934,57 @@ export class BackgroundTaskRegistryClass {export const backgroundTaskRegistry: BackgroundTaskRegistryClass =new BackgroundTaskRegistryClass();+export interface BackgroundTaskShutdownOptions {+ /** `endpoints.agents.backgroundTasks.shutdownInterruptGraceMs`. */+ interruptGraceMs?: number;+ /** Shutdown time left for the drain; defaults to the local coordinator's remaining budget.+ * A clustered worker passes its primary's tighter deadline. */+ getBudgetMs?: () => number | null;+ registry?: BackgroundTaskRegistryClass;+}++/**+ * Registers the graceful-shutdown steps for background tools: stop admitting new tasks+ * while the HTTP server drains, then, after generations are finalized, settle every task+ * whose result is not yet durable.+ */+export function registerBackgroundTaskShutdown(options: BackgroundTaskShutdownOptions = {}): void {+ const registry = options.registry ?? backgroundTaskRegistry;+ const getBudgetMs = options.getBudgetMs ?? getRemainingShutdownMs;+ registerShutdownTask('background task admission', () => registry.closeAdmission(), {+ phase: 'pre-drain',+ priority: 110,+ });+ registerShutdownTask(+ 'background tasks',+ async () => {+ const budgetMs = getBudgetMs();+ if (budgetMs == null) {+ registry.closeAdmission();+ return;+ }+ const summary = await registry.drainForShutdown({+ deadlineAt: Date.now() + Math.max(0, budgetMs - BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS),+ interruptGraceMs:+ options.interruptGraceMs ?? AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT,+ flushReserveMs: BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS,+ reason: BACKGROUND_TASK_SHUTDOWN_MESSAGE,+ });+ if (summary.tracked === 0) {+ return;+ }+ if (summary.unsettled > 0) {+ logger.warn('[background] Shutdown left background task results unconfirmed', summary);+ return;+ }+ logger.info('[background] Drained background tasks for shutdown', summary);+ },+ /** After the generation job manager (100) finalizes interrupted turns, so completion+ * deliveries anchored to them can resolve; before the subagent task store (90). */+ { priority: 95 },+ );+}+/** Content for the synthetic ToolMessage returned when a call is backgrounded. */export function buildBackgroundHandleContent(task: Pick<BackgroundTask, 'id' | 'toolName' | 'status'>,@@ -1830,6 +2020,9 @@ export function buildBackgroundCapacityContent(message = `The server-wide background task registry is at capacity (running limit ${MAX_RUNNING_GLOBAL}). Retry later, or run this call in the foreground.`;} else if (scope === 'global_retention') {message = `The server-wide background task registry is retaining its maximum number of tasks (${MAX_TASKS_GLOBAL}), and pending result processing prevents safe eviction. Retry later, or run this call in the foreground.`;+ } else if (scope === 'shutting_down') {+ message =+ 'This server is shutting down and is not starting new background tasks. Run this call in the foreground, or dispatch it again after the server restarts.';} else if (scope === 'conversation_retention') {message = `This conversation is retaining the maximum number of background tasks (${MAX_TASKS_PER_BUCKET}), and pending result processing prevents safe eviction. Wait for background result processing to finish, or run this call in the foreground.`;} else {diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.tsindex 8aabfefeab..8886ea89c0 100644--- a/packages/api/src/agents/backgroundCompletion.ts+++ b/packages/api/src/agents/backgroundCompletion.ts@@ -6,6 +6,13 @@ export const BACKGROUND_TASK_ABORT_GRACE_MS: number = 60 * 1000;/** Three missed heartbeats prove the process-local executor has been lost. */export const BACKGROUND_TOOL_PRODUCER_LEASE_MS: number = 30_000;export const BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS: number = 10_000;+/** Recorded as the result of a background tool the server had to stop while shutting down. */+export const BACKGROUND_TASK_SHUTDOWN_MESSAGE: string =+ 'Background task was interrupted because its server shut down before it finished. Run it again if you still need the result.';+/** Kept at the end of the background drain for the final durable receipt writes. */+export const BACKGROUND_SHUTDOWN_FLUSH_RESERVE_MS: number = 2_000;+/** Left after the background drain for the shutdown tasks that run after it. */+export const BACKGROUND_SHUTDOWN_TEARDOWN_RESERVE_MS: number = 2_000;/** Host-owned identity recorded before ordinary background tool work begins. */export interface BackgroundToolWakeupRegistration {diff --git a/packages/api/src/agents/handlers.shutdown.spec.ts b/packages/api/src/agents/handlers.shutdown.spec.tsnew file mode 100644index 0000000000..fb9d6d1a00--- /dev/null+++ b/packages/api/src/agents/handlers.shutdown.spec.ts@@ -0,0 +1,350 @@+import { z } from 'zod';+import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';+import type { BackgroundTaskRegistryClass } from './background';+import type * as Handlers from './handlers';+import { BACKGROUND_TASK_SHUTDOWN_MESSAGE } from './backgroundCompletion';++type CreateToolExecuteHandler = typeof Handlers.createToolExecuteHandler;++interface BatchResult {+ content: string;+}++const flushMicrotasks = () => new Promise((resolve) => setImmediate(resolve));++/** Each test drains the registry, which closes admission for the rest of the process,+ * so every test loads its own handler and registry instances. */+function loadModules(): {+ createToolExecuteHandler: CreateToolExecuteHandler;+ registry: BackgroundTaskRegistryClass;+} {+ let loaded:+ | { createToolExecuteHandler: CreateToolExecuteHandler; registry: BackgroundTaskRegistryClass }+ | undefined;+ jest.isolateModules(() => {+ const handlers = jest.requireActual<typeof Handlers>('./handlers');+ const background = jest.requireActual<typeof import('./background')>('./background');+ loaded = {+ createToolExecuteHandler: handlers.createToolExecuteHandler,+ registry: background.backgroundTaskRegistry,+ };+ });+ if (loaded == null) {+ throw new Error('modules failed to load');+ }+ return loaded;+}++const buildConfig = (backgroundToolNames: string[]) => ({+ req: { user: { id: 'shutdown_user' }, body: { conversationId: 'shutdown_convo' } },+ backgroundToolNames,+});++async function runBatch(+ handler: ReturnType<CreateToolExecuteHandler>,+ input: {+ toolCalls: Array<Record<string, unknown>>;+ configurable: Record<string, unknown>;+ metadata: Record<string, unknown>;+ },+): Promise<BatchResult[]> {+ let out: BatchResult[] = [];+ await handler.handle('on_tool_execute', {+ ...input,+ agentId: 'agent_shutdown',+ resolve: (results: BatchResult[]) => {+ out = results;+ },+ reject: (error: Error) => {+ throw error;+ },+ } as unknown as Parameters<typeof handler.handle>[1]);+ return out;+}++function completionAdapter() {+ const persistResult = jest.fn(async () => true);+ const retire = jest.fn(async () => true);+ const renew = jest.fn(async () => true);+ const persist = jest.fn(async () => true);+ return {+ persistResult,+ retire,+ renew,+ persist,+ backgroundToolCompletion: {+ preregister: jest.fn(async () => ({ renew, persistResult, retire })),+ persist,+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),+ },+ };+}++/** A search tool whose invocation the test controls; it may honor or ignore the abort. */+function controlledTool(options: { honorAbort: boolean }) {+ let resolveInvocation: (value: { content: string }) => void = () => undefined;+ const tool = {+ name: 'search_mcp_docs',+ description: 'search docs',+ schema: z.object({ q: z.string() }),+ invoke: jest.fn(+ (_input: unknown, config?: { signal?: AbortSignal }) =>+ new Promise((resolve, reject) => {+ resolveInvocation = resolve;+ if (!options.honorAbort) {+ return;+ }+ config?.signal?.addEventListener(+ 'abort',+ () => reject(new DOMException('This operation was aborted', 'AbortError')),+ { once: true },+ );+ }),+ ),+ } as unknown as StructuredToolInterface;+ return { tool, resolve: (content: string) => resolveInvocation({ content }) };+}++const drainOptions = () => ({+ deadlineAt: Date.now() + 800,+ interruptGraceMs: 300,+ flushReserveMs: 100,+ reason: BACKGROUND_TASK_SHUTDOWN_MESSAGE,+});++const interruptedOutput = `Error: [search_mcp_docs] tool call failed: ${BACKGROUND_TASK_SHUTDOWN_MESSAGE}`;++describe('createToolExecuteHandler — background tasks at shutdown', () => {+ it('interrupts a running tool and delivers the shutdown reason as its result', async () => {+ const { createToolExecuteHandler, registry } = loadModules();+ const adapter = completionAdapter();+ const { tool } = controlledTool({ honorAbort: true });+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: adapter.backgroundToolCompletion,+ });++ const [dispatch] = await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-interrupt',+ name: tool.name,+ args: { q: 'slow', run_in_background: true },+ stepId: 'step-interrupt',+ },+ ],+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-interrupt' },+ });+ const taskId = JSON.parse(dispatch.content).background_task_id;++ const summary = await registry.drainForShutdown(drainOptions());++ expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 0, unsettled: 0 });+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);+ expect(adapter.persistResult).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'error', output: interruptedOutput }),+ );+ expect(adapter.retire).not.toHaveBeenCalled();+ expect(registry.get('shutdown_user', 'shutdown_convo', taskId)).toMatchObject({+ status: 'error',+ error: BACKGROUND_TASK_SHUTDOWN_MESSAGE,+ });+ });++ it('records an interrupted result for a tool that ignores the abort, and a late result keeps it', async () => {+ const { createToolExecuteHandler, registry } = loadModules();+ const adapter = completionAdapter();+ const controlled = controlledTool({ honorAbort: false });+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [controlled.tool] }),+ backgroundToolCompletion: adapter.backgroundToolCompletion,+ });++ const [dispatch] = await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-stubborn',+ name: controlled.tool.name,+ args: { q: 'stubborn', run_in_background: true },+ stepId: 'step-stubborn',+ },+ ],+ configurable: buildConfig([controlled.tool.name]),+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-stubborn' },+ });+ const taskId = JSON.parse(dispatch.content).background_task_id;++ const summary = await registry.drainForShutdown(drainOptions());++ expect(summary).toEqual({ tracked: 1, interrupted: 1, flushed: 1, unsettled: 0 });+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);+ expect(adapter.persistResult).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'error', output: interruptedOutput }),+ );+ expect(registry.get('shutdown_user', 'shutdown_convo', taskId)?.status).toBe('error');++ adapter.persist.mockResolvedValueOnce(false);+ controlled.resolve('late result');+ await flushMicrotasks();+ await flushMicrotasks();++ expect(adapter.persistResult).toHaveBeenCalledTimes(1);+ expect(adapter.retire).not.toHaveBeenCalled();+ });++ it('stores a finished code result durably when its harvest is still waiting', async () => {+ const { createToolExecuteHandler, registry } = loadModules();+ const adapter = completionAdapter();+ const codeTool = {+ name: 'execute_code',+ description: 'run code',+ schema: z.object({ lang: z.string(), code: z.string() }),+ invoke: jest.fn(async () => ({ content: 'stdout:\nhello' })),+ } as unknown as StructuredToolInterface;+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [codeTool] }),+ backgroundToolCompletion: adapter.backgroundToolCompletion,+ persistBackgroundCodeResult: () => new Promise(() => undefined),+ });++ await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-code-harvest',+ name: 'execute_code',+ args: { lang: 'py', code: 'print(1)', run_in_background: true },+ stepId: 'step-code-harvest',+ turn: 1,+ },+ ],+ configurable: buildConfig(['execute_code']),+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-code-harvest' },+ });+ await flushMicrotasks();+ await flushMicrotasks();+ expect(adapter.persistResult).not.toHaveBeenCalled();++ const summary = await registry.drainForShutdown(drainOptions());++ expect(summary).toEqual({ tracked: 1, interrupted: 0, flushed: 1, unsettled: 0 });+ expect(adapter.persistResult).toHaveBeenCalledTimes(1);+ expect(adapter.persistResult).toHaveBeenCalledWith(+ expect.objectContaining({ status: 'completed', output: expect.stringContaining('hello') }),+ );+ });++ it('waits for the projected result when an independent receipt write returns false', async () => {+ const { createToolExecuteHandler, registry } = loadModules();+ const adapter = completionAdapter();+ adapter.persistResult.mockResolvedValue(false);+ let resolveProjection: (ready: boolean) => void = () => undefined;+ adapter.persist.mockImplementation(+ () =>+ new Promise<boolean>((resolve) => {+ resolveProjection = resolve;+ }),+ );+ const tool = {+ name: 'search_mcp_docs',+ description: 'search docs',+ schema: z.object({ q: z.string() }),+ invoke: jest.fn(async () => ({ content: 'projected output' })),+ } as unknown as StructuredToolInterface;+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: adapter.backgroundToolCompletion,+ });++ await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-projected',+ name: tool.name,+ args: { q: 'projected', run_in_background: true },+ stepId: 'step-projected',+ },+ ],+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-projected' },+ });+ await flushMicrotasks();+ expect(adapter.persistResult).toHaveReturnedTimes(1);+ expect(adapter.persist).toHaveBeenCalledTimes(1);++ const draining = registry.drainForShutdown(drainOptions());+ resolveProjection(true);+ expect(await draining).toEqual({ tracked: 1, interrupted: 0, flushed: 0, unsettled: 0 });+ expect(adapter.retire).not.toHaveBeenCalled();+ });++ it('reports an unconfirmed result when neither receipt nor projection becomes durable', async () => {+ const { createToolExecuteHandler, registry } = loadModules();+ const adapter = completionAdapter();+ adapter.persistResult.mockResolvedValue(false);+ adapter.persist.mockResolvedValue(false);+ const tool = {+ name: 'search_mcp_docs',+ description: 'search docs',+ schema: z.object({ q: z.string() }),+ invoke: jest.fn(async () => ({ content: 'not durable' })),+ } as unknown as StructuredToolInterface;+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: adapter.backgroundToolCompletion,+ });++ await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-undurable',+ name: tool.name,+ args: { q: 'undurable', run_in_background: true },+ stepId: 'step-undurable',+ },+ ],+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-undurable' },+ });+ await flushMicrotasks();+ expect(adapter.retire).toHaveBeenCalledTimes(1);++ expect(await registry.drainForShutdown(drainOptions())).toEqual({+ tracked: 1,+ interrupted: 0,+ flushed: 1,+ unsettled: 1,+ });+ });++ it('refuses a new background dispatch once shutdown closes admission', async () => {+ const { createToolExecuteHandler, registry } = loadModules();+ const adapter = completionAdapter();+ const { tool } = controlledTool({ honorAbort: true });+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: adapter.backgroundToolCompletion,+ });+ registry.closeAdmission();++ const [dispatch] = await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-after-shutdown',+ name: tool.name,+ args: { q: 'late', run_in_background: true },+ stepId: 'step-after-shutdown',+ },+ ],+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'shutdown_convo', run_id: 'response-after-shutdown' },+ });++ expect(JSON.parse(dispatch.content)).toMatchObject({+ status: 'rejected',+ scope: 'shutting_down',+ });+ expect(tool.invoke).not.toHaveBeenCalled();+ expect(adapter.backgroundToolCompletion.preregister).not.toHaveBeenCalled();+ });+});diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.tsindex 969e886c1b..ab892079b7 100644--- a/packages/api/src/agents/handlers.ts+++ b/packages/api/src/agents/handlers.ts@@ -96,6 +96,11 @@ import {isCodeSessionToolName,isFileResourceToolName,} from './tools';+import {+ BACKGROUND_TASK_ABORT_GRACE_MS,+ BACKGROUND_TASK_SHUTDOWN_MESSAGE,+ BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,+} from './backgroundCompletion';import {createCodeApiRateLimitBudget,isAbortError,@@ -109,10 +114,6 @@ import {contentFilterModelBoundBlockResponse,isContentFilterError,} from '~/middleware/contentFilter';-import {- BACKGROUND_TASK_ABORT_GRACE_MS,- BACKGROUND_TOOL_PRODUCER_HEARTBEAT_MS,-} from './backgroundCompletion';import {WorkspaceToolHttpError,WORKSPACE_EDIT_MAX_COUNT,@@ -5888,7 +5889,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand};}const backgroundAbortController = new AbortController();- let backgroundAbortSource: 'manual' | 'timeout' | undefined;+ let backgroundAbortSource: 'manual' | 'timeout' | 'shutdown' | undefined;const created = backgroundTaskRegistry.create({...(detachedReservation?.status === 'reserved'? { taskId: detachedReservation.taskId }@@ -5959,6 +5960,62 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand);}}+ let durableReceiptWrite: Promise<boolean> | undefined;+ let durableReceiptAmbiguous = false;+ let durableResultConfirmed = false;+ let resolveDurableReceipt: () => void = () => undefined;+ const durableReceiptSettled = new Promise<void>((resolve) => {+ resolveDurableReceipt = resolve;+ });+ const confirmDurableResult = (): void => {+ durableResultConfirmed = true;+ resolveDurableReceipt();+ };+ /** One durable receipt per task. The task's own settlement and a+ * shutdown flush share the first write, so neither can retire or+ * contradict a receipt the other already stored. */+ const writeDurableReceipt = (receipt: {+ status: 'completed' | 'error' | 'cancelled';+ output?: string;+ settledAt: Date;+ }): Promise<boolean> => {+ if (durableReceiptWrite != null) {+ return durableReceiptWrite;+ }+ durableReceiptWrite = (async (): Promise<boolean> => {+ if (completionAdmission?.persistResult == null) {+ return false;+ }+ try {+ return await completionAdmission.persistResult({+ status: receipt.status,+ output: truncateMiddle(+ receipt.output ?? '',+ backgroundCompletionResultMaxChars,+ ),+ settledAt: receipt.settledAt,+ });+ } catch (receiptError) {+ durableReceiptAmbiguous = true;+ logger.warn(+ `[background] Failed to persist independent result receipt for task ${task.id}:`,+ receiptError,+ );+ return false;+ }+ })();+ void durableReceiptWrite.then((written) => {+ if (written) {+ confirmDurableResult();+ }+ });+ return durableReceiptWrite;+ };+ /** The task's own terminal result, recorded before its (possibly slow)+ * persistence starts, so a shutdown flush can store it durably. */+ let settledReceipt:+ | { status: 'completed' | 'error' | 'cancelled'; output?: string }+ | undefined;/** Persists the settled result onto the dispatch turn's message* (patch the tool-call part's output, persist generated files,* append attachments), so a backgrounded code call reads like a@@ -6021,37 +6078,18 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand);const backgroundTask = resolveBackgroundTask();let durableReceiptReady = false;- let durableReceiptAmbiguous = false;- const persistDurableReceipt = async (receipt: {+ const persistDurableReceipt = (receipt: {status: 'completed' | 'error' | 'cancelled';output?: string;- }): Promise<boolean> => {- if (completionAdmission?.persistResult == null) {- return false;- }- try {- return await completionAdmission.persistResult({- status: receipt.status,- output: truncateMiddle(- receipt.output ?? '',- backgroundCompletionResultMaxChars,- ),- settledAt: backgroundTask.settledAt,- });- } catch (receiptError) {- durableReceiptAmbiguous = true;- logger.warn(- `[background] Failed to persist independent result receipt for task ${task.id}:`,- receiptError,- );- return false;- }- };+ }): Promise<boolean> =>+ writeDurableReceipt({ ...receipt, settledAt: backgroundTask.settledAt });const retireFailedPersistence = async (reason: string,certainty: 'definite' | 'ambiguous',): Promise<void> => {- if (durableReceiptAmbiguous) {+ /** Never retire a delivery that already holds a durable receipt,+ * including one a shutdown flush stored ahead of this path. */+ if (durableReceiptAmbiguous || (await durableReceiptWrite) === true) {return;}if (completionAdmission == null) {@@ -6136,6 +6174,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand'definite',);} else if (deliveryReady && !durableReceiptReady) {+ confirmDurableResult();completionAdmission?.expedite?.();}} catch (persistError) {@@ -6207,6 +6246,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandoutput: params.output ?? localTask?.result,});}+ if (persisted.deliveryReady !== false) {+ confirmDurableResult();+ }if (persisted.deliveryReady === false) {await retireFailedPersistence('background code result was not persisted',@@ -6269,6 +6311,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandartifact?: unknown;status: 'completed' | 'error' | 'cancelled';}): Promise<void> => {+ settledReceipt = { status: params.status, output: params.output };/** Held for the whole persist, including a code harvest that waits for* a long dispatch turn, so retention pressure cannot evict the task. */backgroundTaskRegistry.markCompletionPersistencePending(@@ -6431,7 +6474,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand}, BACKGROUND_TASK_ABORT_GRACE_MS);producerRetirementTimeout.unref?.();};- void (async () => {+ const settlement = (async () => {try {const result = await withBackgroundTaskTimeout(invokePromise,@@ -6552,7 +6595,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandbackgroundControlEnabled,),);- const errorOutput = policyError ?? message;+ /** Tools report an abort in their own words; a shutdown interrupt+ * tells the agent why it happened and that it may retry. */+ const errorOutput =+ backgroundAbortSource === 'shutdown'+ ? BACKGROUND_TASK_SHUTDOWN_MESSAGE+ : (policyError ?? message);const filteredError =policyError == null? filteredToolOutputResult(tc, backgroundReq, {@@ -6621,6 +6669,64 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandawait stopProducerHeartbeat();}})();+ void settlement.then(+ () => {+ if (completionAdmission?.persistResult == null || !completionPreregistered) {+ resolveDurableReceipt();+ }+ },+ () => undefined,+ );+ backgroundTaskRegistry.trackShutdown(task, {+ settled: durableReceiptSettled,+ interrupt: (reason) => {+ if (backgroundAbortSource != null || backgroundAbortController.signal.aborted) {+ return;+ }+ backgroundAbortSource = 'shutdown';+ backgroundAbortController.abort(new DOMException(reason, 'AbortError'));+ },+ flush: async (reason) => {+ await stopProducerHeartbeat();+ if (durableReceiptWrite != null) {+ await durableReceiptWrite;+ } else if (settledReceipt != null) {+ await writeDurableReceipt({ ...settledReceipt, settledAt: new Date() });+ } else {+ const failure = toBackgroundToolFailure(tc.name, reason);+ backgroundTaskRegistry.fail(+ backgroundUserId,+ backgroundConversationId,+ task.id,+ isCodeCall ? failure : reason,+ { harvestStarted: harvestEnabled },+ );+ try {+ await persistDetachedTerminal({ status: 'failed', error: failure });+ } catch (detachedError) {+ logger.warn(+ `[background] Failed to settle detached action for interrupted task ${task.id}:`,+ detachedError,+ );+ }+ await writeDurableReceipt({+ status: 'error',+ output: failure,+ settledAt: new Date(),+ });+ }+ if (durableResultConfirmed || completionAdmission?.persistResult == null) {+ return;+ }+ if (settledReceipt != null) {+ await settlement;+ if (durableResultConfirmed) {+ return;+ }+ }+ throw new Error(`Background task ${task.id} has no durable shutdown result`);+ },+ });if (detachedReservation?.status === 'reserved' &&eventActorDetachedAction != null &&diff --git a/packages/api/src/agents/index.ts b/packages/api/src/agents/index.tsindex 6b0cfc3a3c..5869bdf582 100644--- a/packages/api/src/agents/index.ts+++ b/packages/api/src/agents/index.ts@@ -83,7 +83,7 @@ export * from './activityPhases';export * from './subagentDelivery';export * from './view';export * from './tasks';-export { backgroundTaskRegistry } from './background';+export { backgroundTaskRegistry, registerBackgroundTaskShutdown } from './background';export * from './workspace';export * from './reasoningLabels';export * from './refusal';diff --git a/packages/api/src/agents/triggers/service.delivery.spec.ts b/packages/api/src/agents/triggers/service.delivery.spec.tsindex 43486a54b6..c62b09e8ce 100644--- a/packages/api/src/agents/triggers/service.delivery.spec.ts+++ b/packages/api/src/agents/triggers/service.delivery.spec.ts@@ -292,6 +292,39 @@ describe('durable agent trigger service', () => {await service.stop();});+ it('keeps background producer writes available after the delivery engine stops for shutdown', async () => {+ const methods = deliveryMethods({+ persistAgentBackgroundToolResult: jest.fn(async () => true),+ renewAgentTriggerDeliveryProducerLease: jest.fn(async () => true),+ expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 1, held: 0 })),+ });+ const service = createAgentTriggerService({ methods });+ await service.initialize({ address });+ await service.stop();++ await expect(+ service.renewProducerLease(+ 'trigger_background',+ 'background-tool-completion',+ new Date(START.getTime() + 30_000),+ ),+ ).resolves.toBe(true);+ await expect(+ service.persistBackgroundToolResult({+ deliveryKey: 'trigger_background',+ sourceId: 'background-tool-completion',+ result,+ }),+ ).resolves.toBe(true);+ await flush();++ expect(methods.persistAgentBackgroundToolResult).toHaveBeenCalledTimes(1);+ expect(methods.expediteAgentTriggerDeliveries).not.toHaveBeenCalled();+ await expect(service.enqueue(envelope())).rejects.toBeInstanceOf(+ AgentTriggerServiceUnavailableError,+ );+ });+it("expedites a principal's completion deliveries when one of its generations settles", async () => {const methods = deliveryMethods({expediteAgentTriggerDeliveries: jest.fn(async () => ({ expedited: 2, held: 0 })),diff --git a/packages/api/src/agents/triggers/service.ts b/packages/api/src/agents/triggers/service.tsindex 53de18fc35..f2786ed860 100644--- a/packages/api/src/agents/triggers/service.ts+++ b/packages/api/src/agents/triggers/service.ts@@ -783,9 +783,12 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A}return retired;}),+ // A background tool keeps running while the server drains, and its result must+ // still reach the durable receipt, so producer writes stay available after+ // admissions and the delivery engine stop.renewProducerLease: (deliveryKey, sourceId, leaseUntil) =>runAsSystem(async () =>- requireMethods().renewAgentTriggerDeliveryProducerLease({+ requireCleanupMethods().renewAgentTriggerDeliveryProducerLease({deliveryKey,sourceId,leaseUntil,@@ -793,7 +796,7 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A),persistBackgroundToolResult: (input) =>runAsSystem(async () => {- const persist = requireMethods().persistAgentBackgroundToolResult;+ const persist = requireCleanupMethods().persistAgentBackgroundToolResult;const persisted = persist == null ? false : await persist(input);if (persisted) {expediteCompletions({ deliveryKeys: [input.deliveryKey] });diff --git a/packages/api/src/app/shutdown.spec.ts b/packages/api/src/app/shutdown.spec.tsindex 3b1a4631e9..7ede31b3bc 100644--- a/packages/api/src/app/shutdown.spec.ts+++ b/packages/api/src/app/shutdown.spec.ts@@ -4,6 +4,7 @@ import {setupGracefulShutdown,isShutdownInProgress,registerShutdownTask,+ getClusterShutdownBudgetMs,__resetShutdownStateForTests,} from './shutdown';@@ -18,6 +19,52 @@ const triggerSignal = (signal: NodeJS.Signals): void => {const flush = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));+describe('getClusterShutdownBudgetMs', () => {+ it('prefers the primary force-exit deadline over the worker timer', () => {+ expect(+ getClusterShutdownBudgetMs({+ deadlineAt: 14_000,+ forceExitMs: 30_000,+ remainingMs: 60_000,+ elapsedMs: 1_000,+ now: 10_000,+ }),+ ).toBe(4_000);+ });++ it('uses the elapsed estimate when the primary deadline was not received', () => {+ expect(+ getClusterShutdownBudgetMs({+ deadlineAt: null,+ forceExitMs: 30_000,+ remainingMs: 60_000,+ elapsedMs: 2_000,+ }),+ ).toBe(28_000);+ });++ it('honors the tighter local limit and requires an active shutdown', () => {+ expect(+ getClusterShutdownBudgetMs({+ deadlineAt: 15_000,+ forceExitMs: 30_000,+ remainingMs: 1_000,+ elapsedMs: 5_000,+ now: 10_000,+ }),+ ).toBe(1_000);+ expect(+ getClusterShutdownBudgetMs({+ deadlineAt: 15_000,+ forceExitMs: 30_000,+ remainingMs: null,+ elapsedMs: 5_000,+ now: 10_000,+ }),+ ).toBeNull();+ });+});+describe('setupGracefulShutdown', () => {let server: http.Server;let exitSpy: jest.SpyInstance;diff --git a/packages/api/src/app/shutdown.ts b/packages/api/src/app/shutdown.tsindex 63216a1053..7c56981565 100644--- a/packages/api/src/app/shutdown.ts+++ b/packages/api/src/app/shutdown.ts@@ -68,6 +68,25 @@ export function getRemainingShutdownMs(): number | null {return Math.max(0, SHUTDOWN_TIMEOUT_MS - (Date.now() - shutdownStartedAt));}+export function getClusterShutdownBudgetMs({+ deadlineAt,+ forceExitMs,+ remainingMs = getRemainingShutdownMs(),+ elapsedMs = getShutdownElapsedMs(),+ now = Date.now(),+}: {+ deadlineAt: number | null;+ forceExitMs: number;+ remainingMs?: number | null;+ elapsedMs?: number | null;+ now?: number;+}): number | null {+ if (remainingMs == null || elapsedMs == null) {+ return null;+ }+ return Math.min(remainingMs, deadlineAt == null ? forceExitMs - elapsedMs : deadlineAt - now);+}+export function isShutdownInProgress(): boolean {return isShuttingDown;}diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.tsindex 7ff73ec7a8..5e0eead7ce 100644--- a/packages/data-provider/src/config.spec.ts+++ b/packages/data-provider/src/config.spec.ts@@ -771,6 +771,7 @@ describe('agent background task config', () => {completionWakeups: true,completionResultMaxChars: 24 * 1024,ordinaryToolCancellation: false,+ shutdownInterruptGraceMs: 5_000,});});@@ -789,6 +790,7 @@ describe('agent background task config', () => {completionWakeups: false,completionResultMaxChars: 24 * 1024,ordinaryToolCancellation: false,+ shutdownInterruptGraceMs: 5_000,});});@@ -807,9 +809,31 @@ describe('agent background task config', () => {completionWakeups: true,completionResultMaxChars: 24 * 1024,ordinaryToolCancellation: true,+ shutdownInterruptGraceMs: 5_000,});});+ it('accepts a bounded shutdown interrupt grace', () => {+ const result = configSchema.safeParse({+ version: '1.0',+ endpoints: { agents: { backgroundTasks: { shutdownInterruptGraceMs: 0 } } },+ });++ expect(result.success).toBe(true);+ if (result.success) {+ expect(result.data.endpoints?.agents?.backgroundTasks?.shutdownInterruptGraceMs).toBe(0);+ }+ });++ it.each([-1, 60_001, 1.5])('rejects an unsafe shutdown interrupt grace: %s', (graceMs) => {+ expect(+ configSchema.safeParse({+ version: '1.0',+ endpoints: { agents: { backgroundTasks: { shutdownInterruptGraceMs: graceMs } } },+ }).success,+ ).toBe(false);+ });+it('accepts a bounded durable completion result limit', () => {const result = configSchema.safeParse({version: '1.0',diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex d5b8eb0185..7afb9cb280 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -25,6 +25,7 @@ import {export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;+export const AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT = 5_000;import {MAX_SUBAGENTS,MAX_SUBAGENTS_CEILING,@@ -1575,6 +1576,15 @@ export const agentsEndpointSchema = baseEndpointSchema/** Cooperative cancellation for process-local ordinary tools. Off* by default so existing deployments opt into the new control. */ordinaryToolCancellation: z.boolean().optional().default(false),+ /** During graceful shutdown, how long an interrupted background tool gets to+ * settle on its own before its result is recorded as interrupted. */+ shutdownInterruptGraceMs: z+ .number()+ .int()+ .min(0)+ .max(60_000)+ .optional()+ .default(AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT),}).optional(),skills: z