Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🗳️ fix: Recover Durable Background Tool Receipts (#15752)
Danny Avilaa353e117bc3c
diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.tsindex 1be3ed6c71..b50969d24c 100644--- a/packages/api/src/agents/background.spec.ts+++ b/packages/api/src/agents/background.spec.ts@@ -2096,7 +2096,7 @@ describe('runCheckBackgroundTask (singleton)', () => {);});- it('returns a same-generation result through a local claim that persistence can preserve', async () => {+ it('lets a later poll collect a receipt without inheriting an abandoned local claim', async () => {const created = backgroundTaskRegistry.create({userId: 'claim_user',conversationId: 'claim_convo',@@ -2119,25 +2119,33 @@ describe('runCheckBackgroundTask (singleton)', () => {.fn().mockResolvedValueOnce({ status: 'not_ready' }).mockResolvedValueOnce({ status: 'not_ready' })+ .mockResolvedValueOnce({ status: 'not_ready' }).mockResolvedValueOnce({ status: 'acquired', results: [] });const request = {userId: 'claim_user',conversationId: 'claim_convo',args: { background_task_id: created.task.id },- toolCallId: 'poll-call',agentId: 'agent_parent_1',runId: 'poll-run',+ generationId: 'response-claim',claimBackgroundToolResult,};- const persisting = JSON.parse(await runCheckBackgroundTask(request));+ const persisting = JSON.parse(+ await runCheckBackgroundTask({ ...request, toolCallId: 'poll-call-1' }),+ );expect(persisting).toMatchObject({ status: 'result_persisting' });expect(JSON.stringify(persisting)).not.toContain('CLAIMED RESULT');- const replay = JSON.parse(await runCheckBackgroundTask(request));- expect(replay).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });expect(backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,- ).toMatchObject({ kind: 'manual' });+ ).toBeUndefined();+ const laterPoll = JSON.parse(+ await runCheckBackgroundTask({ ...request, toolCallId: 'poll-call-2' }),+ );+ expect(laterPoll).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });+ expect(+ backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,+ ).toMatchObject({ kind: 'manual', generationId: 'response-claim' });expect(retire).toHaveBeenCalledTimes(1);expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {onlyIfUnclaimed: true,@@ -2147,10 +2155,68 @@ describe('runCheckBackgroundTask (singleton)', () => {messageId: 'response-claim',taskId: created.task.id,kind: 'manual',+ generationId: 'response-claim',+ allowUnfinished: true,}),);});+ it('reclaims a dead manual delivery after its owning generation is gone', async () => {+ const claimBackgroundToolResult = jest+ .fn()+ .mockResolvedValueOnce({+ status: 'claimed',+ messageId: 'response-recovered',+ claim: {+ kind: 'manual',+ claimId: 'abandoned-poll',+ generationId: 'response-abandoned',+ },+ })+ .mockResolvedValueOnce({+ status: 'acquired',+ messageId: 'response-recovered',+ results: [+ {+ taskId: 'task-recovered-manual',+ toolCallId: 'call-recovered-manual',+ toolName: 'mutating_tool',+ status: 'completed',+ output: 'mutation already completed',+ },+ ],+ });+ const recoverDeadBackgroundToolClaim = jest.fn(async () => true);++ const result = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'recovered_user',+ conversationId: 'recovered_convo',+ args: { background_task_id: 'task-recovered-manual' },+ toolCallId: 'replacement-poll',+ runId: 'replacement-run',+ generationId: 'response-replacement',+ claimBackgroundToolResult,+ recoverDeadBackgroundToolClaim,+ }),+ );++ expect(result).toMatchObject({+ status: 'completed',+ background_task_id: 'task-recovered-manual',+ result: 'mutation already completed',+ });+ expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith({+ userId: 'recovered_user',+ conversationId: 'recovered_convo',+ messageId: 'response-recovered',+ claimId: 'abandoned-poll',+ kind: 'manual',+ generationId: 'response-abandoned',+ });+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2);+ });+it('delivers a mandatory live-artifact poll without waiting for its dispatch row', async () => {const created = backgroundTaskRegistry.create({userId: 'artifact_poll_user',@@ -2231,6 +2297,68 @@ describe('runCheckBackgroundTask (singleton)', () => {expect(JSON.stringify(result)).not.toContain('PRIVATE UNTIL CONTINUATION');});+ it.each([+ { status: 'completed' as const, field: 'result', output: 'RECOVERED RESULT' },+ { status: 'error' as const, field: 'error', output: 'RECOVERED FAILURE' },+ ])('recovers a durable $status receipt after process-local state is lost', async (terminal) => {+ const claimBackgroundToolResult = jest.fn(async () => ({+ status: 'acquired' as const,+ messageId: 'response-recovered',+ results: [+ {+ taskId: 'task-recovered',+ toolCallId: 'call-recovered',+ toolName: 'slow_tool',+ status: terminal.status,+ output: terminal.output,+ },+ ],+ }));++ const result = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'recovered_user',+ conversationId: 'recovered_convo',+ args: { background_task_id: 'task-recovered' },+ toolCallId: 'recovery-poll',+ runId: 'recovery-run',+ claimBackgroundToolResult,+ }),+ );++ expect(result).toMatchObject({+ status: terminal.status,+ background_task_id: 'task-recovered',+ [terminal.field]: terminal.output,+ });+ expect(claimBackgroundToolResult).toHaveBeenCalledWith(+ expect.not.objectContaining({ messageId: expect.anything() }),+ );+ });++ it('reports a lost executor without implying that a mutating task is safe to retry', async () => {+ const result = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'unknown_user',+ conversationId: 'unknown_convo',+ args: { background_task_id: 'task-unknown' },+ toolCallId: 'unknown-poll',+ runId: 'unknown-run',+ claimBackgroundToolResult: async () => ({+ status: 'outcome_unknown',+ toolName: 'mutating_tool',+ }),+ }),+ );++ expect(result).toMatchObject({+ status: 'outcome_unknown',+ background_task_id: 'task-unknown',+ tool: 'mutating_tool',+ });+ expect(result.message).toContain('Do not repeat a mutating operation automatically');+ });+it('recovers a result through its dead batch-owner claim', async () => {const created = backgroundTaskRegistry.create({userId: 'dead_claim_user',@@ -2256,7 +2384,7 @@ describe('runCheckBackgroundTask (singleton)', () => {status: 'claimed',claim: { kind: 'wakeup', claimId: 'sibling-batch-root' },})- .mockResolvedValueOnce({ status: 'acquired' });+ .mockResolvedValueOnce({ status: 'acquired', results: [] });const recoverDeadBackgroundToolClaim = jest.fn(async () => true);const result = JSON.parse(@@ -2472,6 +2600,9 @@ describe('runCheckBackgroundTask (singleton)', () => {it('polls and one-shot claims a detached subagent result', async () => {const store = new InMemorySubagentTaskStore();const subagentTasks: SubagentTaskConfig = { store, scopeId: 'owner:parent-thread' };+ const claimBackgroundToolResult = jest.fn(async () => {+ throw new Error('message recovery unavailable');+ });const started = store.start({scopeId: subagentTasks.scopeId,idempotencyKey: 'parent-run:parent-agent:call-1',@@ -2494,6 +2625,7 @@ describe('runCheckBackgroundTask (singleton)', () => {conversationId: 'parent-thread',args: { background_task_id: started.task.taskId },subagentTasks,+ claimBackgroundToolResult,}),);expect(first).toEqual(@@ -2512,10 +2644,69 @@ describe('runCheckBackgroundTask (singleton)', () => {conversationId: 'parent-thread',args: { background_task_id: started.task.taskId },subagentTasks,+ claimBackgroundToolResult,}),);expect(second).toEqual(expect.objectContaining({ status: 'claimed', result_claimed: true }));expect(second.result).toBeUndefined();+ expect(claimBackgroundToolResult).not.toHaveBeenCalled();+ });++ it('falls through cleanly when neither background store recognizes a poll id', async () => {+ const store = new InMemorySubagentTaskStore();+ const claimBackgroundToolResult = jest.fn(async () => ({ status: 'not_found' as const }));++ const result = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'owner',+ conversationId: 'parent-thread',+ args: { background_task_id: 'unknown-task' },+ subagentTasks: { store, scopeId: 'owner:parent-thread' },+ claimBackgroundToolResult,+ }),+ );++ expect(result).toEqual({+ status: 'not_found',+ background_task_id: 'unknown-task',+ message: 'No background task with that id exists in this thread.',+ });+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(1);+ });++ it('recovers an ordinary durable result even when the subagent store is unavailable', async () => {+ const store = Object.assign(new InMemorySubagentTaskStore(), {+ claimTask: jest.fn().mockRejectedValue(new SubagentTaskOwnerUnavailableError()),+ });+ const claimBackgroundToolResult = jest.fn(async () => ({+ status: 'acquired' as const,+ results: [+ {+ taskId: 'ordinary-task',+ toolCallId: 'ordinary-call',+ toolName: 'mutating_tool',+ status: 'completed' as const,+ output: 'ordinary result',+ },+ ],+ }));++ const result = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'owner',+ conversationId: 'parent-thread',+ args: { background_task_id: 'ordinary-task' },+ subagentTasks: { store, scopeId: 'owner:parent-thread' },+ claimBackgroundToolResult,+ }),+ );++ expect(result).toMatchObject({+ status: 'completed',+ background_task_id: 'ordinary-task',+ result: 'ordinary result',+ });+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(1);});it('tells a wakeup-enabled parent to yield on an unchanged running subagent', async () => {diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.tsindex 02192fef93..8a5c50a0b0 100644--- a/packages/api/src/agents/background.ts+++ b/packages/api/src/agents/background.ts@@ -48,6 +48,10 @@ import type {SubagentTaskControlResult,SubagentTaskStore,} from '@librechat/agents';+import type {+ BackgroundToolResultClaim,+ BackgroundToolResultRecord,+} from '@librechat/data-schemas';import type { AgentToolOptions } from 'librechat-data-provider';import type { CapabilityToolNames } from './selection';import {@@ -646,6 +650,7 @@ export interface BackgroundTask {kind: 'manual' | 'wakeup';claimId: string;claimedAt: number;+ generationId?: string;};/** The declared tool may return a process-local live artifact. A terminal* same-generation poll may therefore deliver from the local claim after it@@ -653,6 +658,8 @@ export interface BackgroundTask {* finalize and deadlocking that same generation. */liveArtifactPollRequired?: boolean;completionWakeup?: boolean;+ /** The automatic delivery was durably retired before a manual poll took over. */+ completionWakeupRetired?: boolean;/** True while the terminal result is being persisted for automatic delivery. */completionPersistencePending?: boolean;/** Process-local cancellation handle for the preregistered durable delivery.@@ -1552,6 +1559,7 @@ export class BackgroundTaskRegistryClass {const retired = await task.completionWakeupRetire(reason, options);if (retired) {task.completionWakeupRetire = undefined;+ task.completionWakeupRetired = true;task.updatedAt = Date.now();}return retired;@@ -1561,7 +1569,7 @@ export class BackgroundTaskRegistryClass {userId: string,conversationId: string,taskId: string,- claim: { kind: 'manual' | 'wakeup'; claimId: string },+ claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string },): 'acquired' | 'replay' | 'claimed' | 'not_ready' {const task = this.get(userId, conversationId, taskId);if (task == null || task.status === 'running') {@@ -1793,6 +1801,16 @@ function serializeTask(};}+function serializeDurableTask(task: BackgroundToolResultRecord): SerializedBackgroundTask {+ return {+ background_task_id: task.taskId,+ tool: task.toolName,+ status: task.status,+ progress: 1,+ ...(task.status === 'completed' ? { result: task.output } : { error: task.output }),+ };+}+interface SerializedSubagentTask {background_task_id: string;subagent_thread_id?: string;@@ -1949,19 +1967,20 @@ export async function runCheckBackgroundTask(params: {/** Scopes that tool-call id, whose provider ids repeat across runs and agents. */agentId?: string;runId?: string;+ /** Stable response-message identity used to fence abandoned manual claims. */+ generationId?: string;subagentTasks?: SubagentTaskConfig;claimBackgroundToolResult?: (params: {userId: string;conversationId: string;- messageId: string;+ messageId?: string;taskId: string;agentId?: string;kind: 'manual';claimId: string;- }) => Promise<- | { status: 'acquired' | 'not_found' | 'not_ready' }- | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }- >;+ generationId?: string;+ allowUnfinished?: boolean;+ }) => Promise<BackgroundToolResultClaim>;recoverDeadBackgroundToolClaim?: BackgroundToolDeadClaimRecovery;}): Promise<string> {const { userId, conversationId } = params;@@ -2002,13 +2021,19 @@ export async function runCheckBackgroundTask(params: {agentId: task.agentId,kind: 'manual' as const,claimId: invocationId,+ ...(params.generationId == null ? {} : { generationId: params.generationId }),};let durableClaim = await params.claimBackgroundToolResult(durableClaimInput);if (durableClaim.status === 'claimed') {let recovered = false;let recoveryUnavailable = false;+ const existingClaim = durableClaim.claim;+ const recoverableClaim =+ existingClaim?.kind === 'wakeup' ||+ (existingClaim?.kind === 'manual' && existingClaim.generationId != null);if (- durableClaim.claim?.kind === 'wakeup' &&+ recoverableClaim &&+ existingClaim != null &¶ms.recoverDeadBackgroundToolClaim != null) {try {@@ -2016,7 +2041,13 @@ export async function runCheckBackgroundTask(params: {userId,conversationId,messageId: task.messageId,- claimId: durableClaim.claim.claimId,+ claimId: existingClaim.claimId,+ ...(existingClaim.kind === 'manual'+ ? {+ kind: 'manual' as const,+ generationId: existingClaim.generationId,+ }+ : {}),});} catch (error) {recoveryUnavailable = true;@@ -2062,37 +2093,39 @@ export async function runCheckBackgroundTask(params: {* ownership. A live resolver lease wins. Once that resolver is* irreversibly dead-lettered, a dead-only repair reopens the* process-local poll fallback without stealing live work. */- let retired = false;- try {- retired = await backgroundTaskRegistry.retireCompletionWakeup(- userId,- conversationId,- taskId,- 'completion claimed by same-generation manual poll',- { onlyIfUnclaimed: true },- );- if (!retired) {+ let retired = task.completionWakeupRetired === true;+ if (!retired) {+ try {retired = await backgroundTaskRegistry.retireCompletionWakeup(userId,conversationId,taskId,- 'dead completion recovered by same-generation manual poll',- { onlyIfDead: true },+ 'completion claimed by same-generation manual poll',+ { onlyIfUnclaimed: true },);- if (retired) {- localClaimNeedsNoDurableConfirmation = true;- backgroundTaskRegistry.markCompletionPersistenceFailed(+ if (!retired) {+ retired = await backgroundTaskRegistry.retireCompletionWakeup(userId,conversationId,taskId,+ 'dead completion recovered by same-generation manual poll',+ { onlyIfDead: true },);+ if (retired) {+ localClaimNeedsNoDurableConfirmation = true;+ backgroundTaskRegistry.markCompletionPersistenceFailed(+ userId,+ conversationId,+ taskId,+ );+ }}+ } catch (error) {+ logger.warn(+ `[background] Failed to retire automatic completion for manual claim ${taskId}:`,+ error,+ );}- } catch (error) {- logger.warn(- `[background] Failed to retire automatic completion for manual claim ${taskId}:`,- error,- );}if (!retired) {return JSON.stringify({@@ -2102,28 +2135,32 @@ export async function runCheckBackgroundTask(params: {'The task is finished and completion ownership is being settled. Retry this poll shortly.',});}- const localClaim = backgroundTaskRegistry.claimResult(- userId,- conversationId,- taskId,- { kind: 'manual', claimId: invocationId },- );- if (localClaim === 'claimed') {- return JSON.stringify({- status: 'delivery_scheduled',- background_task_id: taskId,- message: 'This result is already assigned to an automatic continuation.',- });- }- if (localClaim === 'not_ready') {- return JSON.stringify({- status: 'result_persisting',- background_task_id: taskId,- message:- 'The task is finished and its result is being made durable. Retry this poll shortly.',- });- }if (task.liveArtifactPollRequired === true) {+ const localClaim = backgroundTaskRegistry.claimResult(+ userId,+ conversationId,+ taskId,+ {+ kind: 'manual',+ claimId: invocationId,+ ...(params.generationId == null ? {} : { generationId: params.generationId }),+ },+ );+ if (localClaim === 'claimed') {+ return JSON.stringify({+ status: 'delivery_scheduled',+ background_task_id: taskId,+ message: 'This result is already assigned to an automatic continuation.',+ });+ }+ if (localClaim === 'not_ready') {+ return JSON.stringify({+ status: 'result_persisting',+ background_task_id: taskId,+ message:+ 'The task is finished and its result is being made durable. Retry this poll shortly.',+ });+ }/** The poll is executing inside the still-unfinished dispatch* generation, so waiting for the durable row would require* that generation to end before it can obey its mandatory@@ -2134,12 +2171,18 @@ export async function runCheckBackgroundTask(params: {localClaimNeedsNoDurableConfirmation = true;}}- /** Ordinary polls do not expose the local result until the durable- * row has copied this manual claim. The owner-process live-artifact- * exception above cannot wait for its own generation to finalize;- * its persister re-reads the local claim after finalization. */+ /** Ordinary polls claim the durable terminal receipt directly.+ * They never reserve a process-local claim while the receipt is+ * absent: a later poll has a different provider tool-call id and+ * could never take over that abandoned reservation. The live-+ * artifact exception above cannot wait for its own generation. */if (!localClaimNeedsNoDurableConfirmation) {- const reconciledClaim = await params.claimBackgroundToolResult(durableClaimInput);+ const reconciledClaim = await params.claimBackgroundToolResult({+ ...durableClaimInput,+ /** Retirement above makes this owner-process takeover safe;+ * the terminal row may still be a mid-turn partial save. */+ allowUnfinished: true,+ });if (reconciledClaim.status === 'claimed') {backgroundTaskRegistry.releaseResultClaim(userId, conversationId, taskId, {kind: 'manual',@@ -2159,6 +2202,25 @@ export async function runCheckBackgroundTask(params: {'The task is finished and its result is being made durable. Retry this poll shortly.',});}+ /** The response's final full save can overwrite a claim stamped+ * on its unfinished partial row. Mirror only a DURABLY acquired+ * claim so the persistence retry re-applies that ownership after+ * finalization; never create an unanchored local reservation. */+ const mirroredClaim = backgroundTaskRegistry.claimResult(+ userId,+ conversationId,+ taskId,+ {+ kind: 'manual',+ claimId: invocationId,+ ...(params.generationId == null ? {} : { generationId: params.generationId }),+ },+ );+ if (mirroredClaim === 'claimed') {+ logger.error(+ `[background] Durable manual claim for ${taskId} conflicts with process-local ownership.`,+ );+ }}}}@@ -2167,20 +2229,145 @@ export async function runCheckBackgroundTask(params: {}const subagentTasks = params.subagentTasks;+ let subagentPollChecked = false;+ let subagentPollError: unknown;+ /** Routed subagents have their own durable/cross-replica store. Resolve+ * them before ordinary-tool Mongo recovery so two fallback reads—or a+ * message-store outage—cannot delay or mask a healthy subagent result. */+ if (action === 'poll' && subagentTasks != null) {+ subagentPollChecked = true;+ try {+ const routedStore = routedSubagentStore(subagentTasks.store);+ const claim =+ routedStore == null+ ? subagentTasks.store.claim(subagentTasks.scopeId, taskId)+ : await routedStore.claimTask(subagentTasks.scopeId, taskId, invocationId);+ const claimed = serializeSubagentClaim(+ claim,+ agentUsesSubagentCompletionWakeups(subagentTasks, params.agentId),+ );+ if (claimed != null) {+ return JSON.stringify(claimed);+ }+ } catch (error) {+ /** This id may still belong to an ordinary background tool whose+ * durable receipt is healthy. Defer the subagent failure until after+ * that independent recovery path has had a chance to identify it. */+ subagentPollError = error;+ }+ }++ if (action === 'poll' && params.claimBackgroundToolResult != null) {+ const durableClaimInput = {+ userId,+ conversationId,+ taskId,+ agentId: params.agentId,+ kind: 'manual' as const,+ claimId: invocationId,+ ...(params.generationId == null ? {} : { generationId: params.generationId }),+ };+ let durableClaim: BackgroundToolResultClaim;+ try {+ durableClaim = await params.claimBackgroundToolResult(durableClaimInput);+ const claimedResult = durableClaim.status === 'claimed' ? durableClaim : undefined;+ const existingClaim = claimedResult?.claim;+ const recoverableClaim =+ existingClaim?.kind === 'wakeup' ||+ (existingClaim?.kind === 'manual' && existingClaim.generationId != null);+ if (+ recoverableClaim &&+ existingClaim != null &&+ claimedResult?.messageId != null &&+ params.recoverDeadBackgroundToolClaim != null+ ) {+ const recovered = await params.recoverDeadBackgroundToolClaim({+ userId,+ conversationId,+ messageId: claimedResult.messageId,+ claimId: existingClaim.claimId,+ ...(existingClaim.kind === 'manual'+ ? {+ kind: 'manual' as const,+ generationId: existingClaim.generationId,+ }+ : {}),+ });+ if (recovered) {+ durableClaim = await params.claimBackgroundToolResult(durableClaimInput);+ }+ }+ } catch (error) {+ logger.warn(`[background] Failed to recover durable task ${taskId} during polling:`, error);+ return JSON.stringify({+ status: 'result_unavailable',+ background_task_id: taskId,+ message:+ 'The durable task receipt is temporarily unavailable. Do not repeat a mutating operation; retry this status check later.',+ });+ }+ if (durableClaim.status === 'acquired') {+ const durableTask = durableClaim.results.find((result) => result.taskId === taskId);+ if (durableTask != null) {+ return JSON.stringify(serializeDurableTask(durableTask));+ }+ return JSON.stringify({+ status: 'result_persisting',+ background_task_id: taskId,+ message: 'The task receipt is settling. Retry this poll shortly.',+ });+ }+ if (durableClaim.status === 'claimed') {+ return JSON.stringify({+ status: 'delivery_scheduled',+ background_task_id: taskId,+ message: 'This result is already assigned to another poll or automatic continuation.',+ });+ }+ if (durableClaim.status === 'outcome_unknown') {+ return JSON.stringify({+ status: 'outcome_unknown',+ background_task_id: taskId,+ tool: durableClaim.toolName,+ message:+ 'The task was launched, but this server cannot confirm a live executor or a durable terminal receipt. Do not repeat a mutating operation automatically; inspect the target system before retrying.',+ });+ }+ if (durableClaim.status === 'not_ready') {+ return JSON.stringify({+ status: 'result_persisting',+ background_task_id: taskId,+ message:+ 'The task exists, but its terminal receipt is not ready. Do not repeat a mutating operation; retry this status check later.',+ });+ }+ }+if (subagentTasks != null) {try {const routedStore = routedSubagentStore(subagentTasks.store);if (action === 'poll') {- const claim =- routedStore == null- ? subagentTasks.store.claim(subagentTasks.scopeId, taskId)- : await routedStore.claimTask(subagentTasks.scopeId, taskId, invocationId);- const claimed = serializeSubagentClaim(- claim,- agentUsesSubagentCompletionWakeups(subagentTasks, params.agentId),- );- if (claimed != null) {- return JSON.stringify(claimed);+ if (!subagentPollChecked) {+ const claim =+ routedStore == null+ ? subagentTasks.store.claim(subagentTasks.scopeId, taskId)+ : await routedStore.claimTask(subagentTasks.scopeId, taskId, invocationId);+ const claimed = serializeSubagentClaim(+ claim,+ agentUsesSubagentCompletionWakeups(subagentTasks, params.agentId),+ );+ if (claimed != null) {+ return JSON.stringify(claimed);+ }+ } else if (subagentPollError != null) {+ if (subagentPollError instanceof SubagentTaskOwnerUnavailableError) {+ return JSON.stringify({+ status: 'unavailable',+ background_task_id: taskId,+ message: subagentPollError.message,+ });+ }+ throw subagentPollError;}} else {const command = buildSubagentControlCommand(args, action);diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.tsindex 914b9d3822..08b84b0b80 100644--- a/packages/api/src/agents/backgroundCompletion.ts+++ b/packages/api/src/agents/backgroundCompletion.ts@@ -46,6 +46,10 @@ export interface BackgroundToolDeadClaimRecoveryInput {conversationId: string;messageId: string;claimId: string;+ /** Omitted for the legacy automatic-wakeup recovery path. */+ kind?: 'manual' | 'wakeup';+ /** Required to prove that a manual claim's owning generation is no longer active. */+ generationId?: string;}export type BackgroundToolDeadClaimRecovery = (diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.tsindex cd79c4ac52..19a4a9560d 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts@@ -220,6 +220,72 @@ describe('background tool completion wakeups', () => {});});+ it('keeps a manual result claim while its owning generation is active', async () => {+ const retire = jest.fn(async () => true);+ const release = jest.fn(async () => true);+ const getGenerationJob = jest.fn(async () => ({+ status: 'running',+ metadata: { responseMessageId: 'response-manual-owner' },+ }));+ const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);+ const recover = createBackgroundToolDeadClaimRecovery(+ retire,+ release,+ getGenerationJob,+ fenceGenerationClaim,+ );++ await expect(+ recover({+ userId: 'user-1',+ conversationId: 'conversation-1',+ messageId: 'response-result',+ claimId: 'manual-poll',+ kind: 'manual',+ generationId: 'response-manual-owner',+ }),+ ).resolves.toBe(false);++ expect(getGenerationJob).toHaveBeenCalledWith('conversation-1');+ expect(release).not.toHaveBeenCalled();+ expect(retire).not.toHaveBeenCalled();+ expect(fenceGenerationClaim).not.toHaveBeenCalled();+ });++ it('releases a manual result claim after its owning generation is gone', async () => {+ const retire = jest.fn(async () => true);+ const release = jest.fn(async () => true);+ const getGenerationJob = jest.fn(async () => null);+ const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);+ const recover = createBackgroundToolDeadClaimRecovery(+ retire,+ release,+ getGenerationJob,+ fenceGenerationClaim,+ );++ await expect(+ recover({+ userId: 'user-1',+ conversationId: 'conversation-1',+ messageId: 'response-result',+ claimId: 'manual-poll',+ kind: 'manual',+ generationId: 'response-manual-owner',+ }),+ ).resolves.toBe(true);++ expect(release).toHaveBeenCalledWith({+ userId: 'user-1',+ conversationId: 'conversation-1',+ messageId: 'response-result',+ kind: 'manual',+ claimId: 'manual-poll',+ });+ expect(retire).not.toHaveBeenCalled();+ expect(fenceGenerationClaim).not.toHaveBeenCalled();+ });+it('does not recover a dead delivery while its admitted generation is still active', async () => {const retire = jest.fn(async () => true);const release = jest.fn(async () => true);diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.tsindex 814f2d319d..03fddfefeb 100644--- a/packages/api/src/agents/backgroundCompletionWakeup.ts+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts@@ -82,6 +82,7 @@ interface GenerationState {status?: unknown;metadata?: {idempotencyClientRequestId?: unknown;+ responseMessageId?: unknown;terminalPersistencePending?: unknown;};}@@ -329,6 +330,12 @@ export function createBackgroundToolCompletionWakeupResolver({if (claim.status === 'claimed') {return { status: 'settled' };}+ if (claim.status === 'outcome_unknown') {+ throw executionError('The process-local background tool outcome is unknown.', {+ code: 'BACKGROUND_TOOL_OUTCOME_UNKNOWN',+ retryable: false,+ });+ }if (claim.status !== 'acquired') {let producerLease: AgentTriggerProducerLeaseStatus;try {@@ -463,7 +470,25 @@ export function createBackgroundToolDeadClaimRecovery(claimId: string;}) => Promise<'fenced' | 'started' | 'unavailable'>,): BackgroundToolDeadClaimRecovery {- return async ({ userId, conversationId, messageId, claimId }) => {+ return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {+ if (kind === 'manual') {+ if (generationId == null || generationId.length === 0) {+ return false;+ }+ const generation = await getGenerationJob(conversationId);+ if (generation?.metadata?.responseMessageId === generationId && isParentActive(generation)) {+ return false;+ }+ /** Releasing a dead manual DELIVERY only re-presents an already durable+ * terminal result. It never retries the completed tool mutation. */+ return releaseClaims({+ userId,+ conversationId,+ messageId,+ kind: 'manual',+ claimId,+ });+ }const claimGenerationIsActive = async (): Promise<boolean> => {const generation = await getGenerationJob(conversationId);return (diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.tsindex b56f795ae6..0e613bc689 100644--- a/packages/api/src/agents/handlers.background.spec.ts+++ b/packages/api/src/agents/handlers.background.spec.ts@@ -2,8 +2,12 @@ import { z } from 'zod';import { logger } from '@librechat/data-schemas';import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';import type { FiltersConfig } from 'librechat-data-provider';+import {+ backgroundTaskRegistry,+ runCheckBackgroundTask,+ CHECK_BACKGROUND_TASK_NAME,+} from './background';import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion';-import { backgroundTaskRegistry, CHECK_BACKGROUND_TASK_NAME } from './background';import { ContentFilterError } from '../middleware/contentFilter';import { createToolExecuteHandler } from './handlers';@@ -110,7 +114,7 @@ describe('createToolExecuteHandler — background tool calls', () => {backgroundToolCompletion: {preregister,persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -173,7 +177,7 @@ describe('createToolExecuteHandler — background tool calls', () => {retire: jest.fn(async () => true),})),persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -205,7 +209,7 @@ describe('createToolExecuteHandler — background tool calls', () => {backgroundToolCompletion: {preregister,persist: jest.fn(async () => true),- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -240,7 +244,7 @@ describe('createToolExecuteHandler — background tool calls', () => {backgroundToolCompletion: {preregister,persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -272,7 +276,7 @@ describe('createToolExecuteHandler — background tool calls', () => {backgroundToolCompletion: {preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),persist: jest.fn(async () => false),- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -295,6 +299,79 @@ describe('createToolExecuteHandler — background tool calls', () => {expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);});+ it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => {+ let finishPersistence: ((persisted: boolean) => void) | undefined;+ const persist = jest.fn(+ () =>+ new Promise<boolean>((resolve) => {+ finishPersistence = resolve;+ }),+ );+ const retire = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false);+ const tool = makeSearchTool({ calls: 0 });+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [tool] }),+ backgroundToolCompletion: {+ preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),+ persist,+ claim: jest.fn(async () => ({ status: 'not_ready' as const })),+ },+ });++ const [dispatch] = await runBatch(handler, {+ toolCalls: [+ {+ id: 'call-poll-before-failure',+ name: tool.name,+ args: { q: 'settle', run_in_background: true },+ stepId: 'step-poll-before-failure',+ },+ ],+ agentId: 'agent_parent_1',+ configurable: buildConfig([tool.name]),+ metadata: { thread_id: 'exec_convo', run_id: 'response-poll-before-failure' },+ });+ await flushMicrotasks();+ const taskId = JSON.parse(dispatch.content).background_task_id as string;++ const waiting = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'exec_user',+ conversationId: 'exec_convo',+ args: { background_task_id: taskId },+ toolCallId: 'manual-poll-1',+ runId: 'response-poll-before-failure',+ claimBackgroundToolResult: async () => ({ status: 'not_ready' }),+ }),+ );+ expect(waiting.status).toBe('result_persisting');+ expect(+ backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId)?.resultClaim,+ ).toBeUndefined();++ finishPersistence?.(false);+ await flushMicrotasks();+ await flushMicrotasks();+ expect(+ backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId)?.completionPersistenceFailed,+ ).toBe(true);++ const recovered = JSON.parse(+ await runCheckBackgroundTask({+ userId: 'exec_user',+ conversationId: 'exec_convo',+ args: { background_task_id: taskId },+ toolCallId: 'manual-poll-2',+ runId: 'response-poll-before-failure',+ claimBackgroundToolResult: async () => ({ status: 'not_ready' }),+ }),+ );+ expect(recovered).toMatchObject({+ status: 'completed',+ result: 'RESULT for settle',+ });+ });+it('keeps durable ownership active when an ambiguous persistence failure cannot retire a lease', async () => {const tool = makeSearchTool({ calls: 0 });const retire = jest.fn(async () => false);@@ -356,7 +433,7 @@ describe('createToolExecuteHandler — background tool calls', () => {backgroundToolCompletion: {preregister,persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -404,7 +481,7 @@ describe('createToolExecuteHandler — background tool calls', () => {retire,})),persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -450,7 +527,7 @@ describe('createToolExecuteHandler — background tool calls', () => {retire: jest.fn(async () => true),})),persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -505,7 +582,7 @@ describe('createToolExecuteHandler — background tool calls', () => {retire: jest.fn(async () => true),})),persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -561,7 +638,7 @@ describe('createToolExecuteHandler — background tool calls', () => {retire,})),persist,- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});@@ -1772,7 +1849,7 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {backgroundToolCompletion: {preregister,persist: jest.fn(async () => true),- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});const configurable = buildConfig(['execute_code']);@@ -2168,7 +2245,7 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {backgroundToolCompletion: {preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),persist: jest.fn(async () => true),- claim: jest.fn(async () => ({ status: 'acquired' as const })),+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),},});const configurable = buildConfig(['execute_code']);diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.tsindex 5dc5a3ae70..8886306f41 100644--- a/packages/api/src/agents/handlers.ts+++ b/packages/api/src/agents/handlers.ts@@ -17,9 +17,9 @@ import type {StreamEventData,ToolEndCallback as SdkToolEndCallback,} from '@librechat/agents';+import type { BackgroundToolResultClaim, ValidationIssue } from '@librechat/data-schemas';import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';import type { CodeEnvRef, PtcToolCallEvent } from 'librechat-data-provider';-import type { ValidationIssue } from '@librechat/data-schemas';import type {WorkspaceEditResult,WorkspacePreviewEditResult,@@ -283,15 +283,14 @@ export interface ToolExecuteOptions {claim: (params: {userId: string;conversationId: string;- messageId: string;+ messageId?: string;taskId: string;agentId?: string;kind: 'manual';claimId: string;- }) => Promise<- | { status: 'acquired' | 'not_found' | 'not_ready' }- | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }- >;+ generationId?: string;+ allowUnfinished?: boolean;+ }) => Promise<BackgroundToolResultClaim>;recoverDeadClaim?: BackgroundToolDeadClaimRecovery;};/** Emits an `attachment` SSE event on the current request's live stream. */@@ -5519,6 +5518,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandkind: current.resultClaim.kind,claimId: current.resultClaim.claimId,claimedAt: new Date(current.resultClaim.claimedAt),+ ...(current.resultClaim.generationId == null+ ? {}+ : { generationId: current.resultClaim.generationId }),},}: {}),@@ -5553,6 +5555,23 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandcertainty === 'ambiguous' ? { onlyIfUnclaimed: true } : undefined,);if (!retired) {+ const current = backgroundTaskRegistry.get(+ backgroundUserId,+ backgroundConversationId,+ task.id,+ );+ /** A prior manual poll may already have durably retired+ * this exact unclaimed delivery. In that case there is+ * no automatic consumer left to race the process-local+ * fallback, even though a second retirement is a no-op. */+ if (current?.completionWakeupRetired === true) {+ backgroundTaskRegistry.markCompletionPersistenceFailed(+ backgroundUserId,+ backgroundConversationId,+ task.id,+ );+ return;+ }logger.warn(`[background] Could not retire failed completion delivery for task ${task.id}.`,);@@ -6071,6 +6090,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandtoolCallId: tc.id,agentId,runId: `${backgroundRunId ?? ''}:${tc.turn ?? ''}`,+ generationId: backgroundRunId,subagentTasks,claimBackgroundToolResult: backgroundToolCompletion?.claim,recoverDeadBackgroundToolClaim: backgroundToolCompletion?.recoverDeadClaim,diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.tsindex 84f67cbfdf..0ebda3db75 100644--- a/packages/api/src/agents/harvest.ts+++ b/packages/api/src/agents/harvest.ts@@ -41,6 +41,7 @@ export interface BackgroundToolResultState {kind: 'manual' | 'wakeup';claimId: string;claimedAt: Date;+ generationId?: string;};}diff --git a/packages/data-provider/src/types/agents.ts b/packages/data-provider/src/types/agents.tsindex cfa7e101bf..e3b4fc8e23 100644--- a/packages/data-provider/src/types/agents.ts+++ b/packages/data-provider/src/types/agents.ts@@ -98,6 +98,8 @@ export namespace Agents {kind: 'manual' | 'wakeup';claimId: string;claimedAt: Date;+ /** Response generation that owns a manual delivery claim. */+ generationId?: string;};};/** The tool call was rejected before execution because its input failed schema validation. */diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.tsindex 32b8978cce..c9c0d02f3b 100644--- a/packages/data-schemas/src/methods/message.spec.ts+++ b/packages/data-schemas/src/methods/message.spec.ts@@ -1118,6 +1118,315 @@ describe('Message Operations', () => {expect(task).not.toHaveProperty('completionWakeup');});+ it('lets a same-generation manual poll claim an anchored terminal receipt', async () => {+ await saveMessage(mockCtx, {+ ...mockMessageData,+ unfinished: true,+ content: [+ {+ type: 'tool_call',+ agentId: 'agent-a',+ tool_call: {+ id: 'call-same-generation',+ name: 'slow_tool',+ output: 'settled output',+ backgroundTask: {+ version: 1,+ taskId: 'task-same-generation',+ toolName: 'slow_tool',+ status: 'completed',+ settledAt: new Date(),+ completionWakeup: true,+ },+ },+ },+ ],+ });++ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ messageId: 'msg123',+ taskId: 'task-same-generation',+ kind: 'wakeup',+ claimId: 'automatic-delivery',+ }),+ ).resolves.toEqual({ status: 'not_ready' });+ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ messageId: 'msg123',+ taskId: 'task-same-generation',+ kind: 'manual',+ claimId: 'manual-poll',+ }),+ ).resolves.toEqual({ status: 'not_ready' });+ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ messageId: 'msg123',+ taskId: 'task-same-generation',+ kind: 'manual',+ claimId: 'manual-poll',+ allowUnfinished: true,+ }),+ ).resolves.toEqual({+ status: 'acquired',+ results: [+ {+ taskId: 'task-same-generation',+ toolCallId: 'call-same-generation',+ toolName: 'slow_tool',+ status: 'completed',+ output: 'settled output',+ agentId: 'agent-a',+ },+ ],+ });+ });++ it('restores same-generation claim ownership after the final response save', async () => {+ const terminalContent = [+ {+ type: 'tool_call',+ tool_call: {+ id: 'call-finalize-race',+ name: 'mutating_tool',+ output: 'mutation completed',+ backgroundTask: {+ version: 1,+ taskId: 'task-finalize-race',+ toolName: 'mutating_tool',+ status: 'completed',+ settledAt: new Date(),+ completionWakeup: true,+ },+ },+ },+ ];+ await saveMessage(mockCtx, {+ ...mockMessageData,+ unfinished: true,+ content: terminalContent,+ });+ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ messageId: 'msg123',+ taskId: 'task-finalize-race',+ kind: 'manual',+ claimId: 'manual-owner',+ generationId: 'response-finalize-owner',+ allowUnfinished: true,+ }),+ ).resolves.toMatchObject({ status: 'acquired' });++ /** The final in-memory response did not observe the concurrent claim and+ * rewrites the content part without it. The detached persistence retry+ * must restore the mirrored owner before the receipt can be replayed. */+ await saveMessage(mockCtx, {+ ...mockMessageData,+ unfinished: false,+ content: terminalContent,+ });+ await updateToolCallResult({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ messageId: 'msg123',+ toolCallId: 'call-finalize-race',+ output: 'mutation completed',+ backgroundTask: {+ taskId: 'task-finalize-race',+ toolName: 'mutating_tool',+ status: 'completed',+ settledAt: new Date(),+ completionWakeup: true,+ resultClaim: {+ kind: 'manual',+ claimId: 'manual-owner',+ claimedAt: new Date(),+ generationId: 'response-finalize-owner',+ },+ },+ });++ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ messageId: 'msg123',+ taskId: 'task-finalize-race',+ kind: 'manual',+ claimId: 'competing-owner',+ }),+ ).resolves.toEqual({+ status: 'claimed',+ claim: {+ kind: 'manual',+ claimId: 'manual-owner',+ generationId: 'response-finalize-owner',+ },+ });+ });++ it('recovers an unclaimed terminal receipt by task id after process-local state is lost', async () => {+ await saveMessage(mockCtx, {+ ...mockMessageData,+ content: [+ {+ type: 'tool_call',+ agentId: 'agent-a',+ tool_call: {+ id: 'call-recovered',+ name: 'slow_tool',+ output: 'durable result',+ backgroundTask: {+ version: 1,+ taskId: 'task-recovered',+ toolName: 'slow_tool',+ status: 'completed',+ settledAt: new Date(),+ },+ },+ },+ ],+ });++ const recovery = {+ status: 'acquired',+ messageId: 'msg123',+ results: [+ {+ taskId: 'task-recovered',+ toolCallId: 'call-recovered',+ toolName: 'slow_tool',+ status: 'completed',+ output: 'durable result',+ agentId: 'agent-a',+ },+ ],+ };+ const recover = () =>+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ taskId: 'task-recovered',+ agentId: 'agent-a',+ kind: 'manual',+ claimId: 'recovery-poll',+ generationId: 'response-recovery-owner',+ });+ await expect(recover()).resolves.toEqual(recovery);+ await expect(recover()).resolves.toEqual(recovery);+ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ taskId: 'task-recovered',+ agentId: 'agent-a',+ kind: 'manual',+ claimId: 'competing-poll',+ }),+ ).resolves.toEqual({+ status: 'claimed',+ messageId: 'msg123',+ claim: {+ kind: 'manual',+ claimId: 'recovery-poll',+ generationId: 'response-recovery-owner',+ },+ });+ await expect(+ claimBackgroundToolResults({+ userId: 'another-user',+ conversationId: mockMessageData.conversationId as string,+ taskId: 'task-recovered',+ kind: 'manual',+ claimId: 'cross-user-poll',+ }),+ ).resolves.toEqual({ status: 'not_found' });+ });++ it('reports an exact durable launch handle as outcome unknown after executor loss', async () => {+ await saveMessage(mockCtx, {+ ...mockMessageData,+ content: [+ {+ type: 'tool_call',+ agentId: 'agent-a',+ tool_call: {+ id: 'call-lost',+ name: 'mutating_tool',+ output: JSON.stringify({+ background_task_id: 'task-lost',+ tool: 'mutating_tool',+ status: 'running',+ message: 'Use check_background_task to poll.',+ }),+ },+ },+ ],+ });++ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ taskId: 'task-lost',+ agentId: 'agent-a',+ kind: 'manual',+ claimId: 'recovery-poll',+ }),+ ).resolves.toEqual({ status: 'outcome_unknown', toolName: 'mutating_tool' });+ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ taskId: 'task-lost',+ agentId: 'agent-b',+ kind: 'manual',+ claimId: 'wrong-agent-poll',+ }),+ ).resolves.toEqual({ status: 'not_found' });+ });++ it('leaves a durable subagent handle to the routed subagent store', async () => {+ await saveMessage(mockCtx, {+ ...mockMessageData,+ content: [+ {+ type: 'tool_call',+ tool_call: {+ id: 'call-subagent',+ name: 'subagent',+ output: JSON.stringify({+ background_task_id: 'task-subagent',+ subagent_thread_id: 'thread-subagent',+ tool: 'subagent',+ subagent_type: 'researcher',+ status: 'running',+ progress: 0,+ }),+ },+ },+ ],+ });++ await expect(+ claimBackgroundToolResults({+ userId: 'user123',+ conversationId: mockMessageData.conversationId as string,+ taskId: 'task-subagent',+ kind: 'manual',+ claimId: 'subagent-poll',+ }),+ ).resolves.toEqual({ status: 'not_found' });+ });+it('claims a terminal result whose persisted claim stamp is a stored null', async () => {/** A full-row save can persist `resultClaim: null`, and the* subfield-preserving settle write keeps it where the old whole-objectdiff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.tsindex b59de98abe..48c9ee3166 100644--- a/packages/data-schemas/src/methods/message.ts+++ b/packages/data-schemas/src/methods/message.ts@@ -466,8 +466,13 @@ export interface BackgroundToolResultRecord {export type BackgroundToolResultClaim =| { status: 'not_found' | 'not_ready' }- | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }- | { status: 'acquired'; results: BackgroundToolResultRecord[] };+ | { status: 'outcome_unknown'; toolName: string }+ | {+ status: 'claimed';+ claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string };+ messageId?: string;+ }+ | { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string };export type SubagentThreadViewMessageRecord = Pick<IMessage,@@ -648,17 +653,23 @@ export interface MessageMethods {kind: 'manual' | 'wakeup';claimId: string;claimedAt: Date;+ generationId?: string;};};}): Promise<{ matched: boolean; unfinished: boolean }>;claimBackgroundToolResults(params: {userId: string;conversationId: string;- messageId: string;+ /** Optional on recovery polls after the process-local task registry was lost. */+ messageId?: string;taskId: string;agentId?: string;kind: 'manual' | 'wakeup';claimId: string;+ /** Response generation that owns this manual result delivery. */+ generationId?: string;+ /** Manual owner-process takeover after automatic delivery was retired. */+ allowUnfinished?: boolean;limit?: number;}): Promise<BackgroundToolResultClaim>;releaseBackgroundToolResultClaims(params: {@@ -1139,6 +1150,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messakind: 'manual' | 'wakeup';claimId: string;claimedAt: Date;+ generationId?: string;};};}): Promise<{ matched: boolean; unfinished: boolean }> {@@ -1334,7 +1346,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messafunction readBackgroundToolResultClaim(row: Pick<IMessage, 'content'>,taskId: string,- ): { kind: 'manual' | 'wakeup'; claimId: string } | undefined {+ ): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined {for (const part of row.content ?? []) {if (part == null || typeof part !== 'object' || Array.isArray(part)) {continue;@@ -1344,7 +1356,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messatool_call?: {backgroundTask?: {taskId?: unknown;- resultClaim?: { kind?: unknown; claimId?: unknown };+ resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown };};};}@@ -1358,7 +1370,13 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messatypeof claim.claimId === 'string' &&claim.claimId.length > 0) {- return { kind: claim.kind, claimId: claim.claimId };+ return {+ kind: claim.kind,+ claimId: claim.claimId,+ ...(typeof claim.generationId === 'string' && claim.generationId.length > 0+ ? { generationId: claim.generationId }+ : {}),+ };}return;}@@ -1417,6 +1435,51 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messareturn results;}+ function readBackgroundToolHandle(+ message: IMessage,+ taskId: string,+ agentId?: string,+ ): string | undefined {+ for (const part of message.content ?? []) {+ if (part == null || typeof part !== 'object' || Array.isArray(part)) {+ continue;+ }+ const record = part as {+ agentId?: unknown;+ tool_call?: { agentId?: unknown; output?: unknown };+ };+ const partAgentId = record.agentId ?? record.tool_call?.agentId;+ const sameAgent =+ agentId == null ||+ partAgentId == null ||+ (typeof partAgentId === 'string' && partAgentId === agentId);+ const output = record.tool_call?.output;+ if (!sameAgent || typeof output !== 'string' || !output.includes(taskId)) {+ continue;+ }+ try {+ const handle = JSON.parse(output) as {+ background_task_id?: unknown;+ subagent_type?: unknown;+ tool?: unknown;+ status?: unknown;+ };+ if (+ handle.background_task_id === taskId &&+ handle.status === 'running' &&+ typeof handle.subagent_type !== 'string' &&+ typeof handle.tool === 'string' &&+ handle.tool.length > 0+ ) {+ return handle.tool;+ }+ } catch {+ continue;+ }+ }+ return;+ }+/** Atomically elects manual polling or one automatic continuation. Wakeups* also claim a bounded set of already-settled siblings from the same parent* response, avoiding one paid continuation per concurrently completed tool. */@@ -1428,39 +1491,91 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): MessaagentId,kind,claimId,+ generationId,+ allowUnfinished = false,limit = kind === 'wakeup' ? MAX_BACKGROUND_TOOL_RESULT_BATCH : 1,}: {userId: string;conversationId: string;- messageId: string;+ messageId?: string;taskId: string;agentId?: string;kind: 'manual' | 'wakeup';claimId: string;+ generationId?: string;+ allowUnfinished?: boolean;limit?: number;}): Promise<BackgroundToolResultClaim> {+ const requestedMessageId = messageId?.trim();+ const requestedGenerationId = generationId?.trim();if (- messageId.length === 0 ||- messageId.length > 256 ||+ (requestedMessageId != null &&+ (requestedMessageId.length === 0 || requestedMessageId.length > 256)) ||taskId.length === 0 ||taskId.length > 256 ||claimId.length === 0 ||claimId.length > 128 ||+ (requestedGenerationId != null &&+ (requestedGenerationId.length === 0 || requestedGenerationId.length > 256)) ||+ (requestedGenerationId != null && kind !== 'manual') ||+ (allowUnfinished && kind !== 'manual') ||(kind !== 'manual' && kind !== 'wakeup')) {throw new TypeError('Invalid background tool result claim');}const boundedLimit = Math.max(1, Math.min(MAX_BACKGROUND_TOOL_RESULT_BATCH, limit));const Message = mongoose.models.Message as Model<IMessage>;- const row = await Message.findOne({ user: userId, conversationId, messageId })- .select({ content: 1, unfinished: 1 })+ const row = await Message.findOne({+ user: userId,+ conversationId,+ ...(requestedMessageId != null+ ? { messageId: requestedMessageId }+ : {+ content: {+ $elemMatch: { 'tool_call.backgroundTask.taskId': taskId },+ },+ }),+ })+ .select({ content: 1, unfinished: 1, messageId: 1 })+ .sort({ createdAt: -1, _id: -1 }).lean<IMessage | null>();if (row == null) {+ if (requestedMessageId == null) {+ const escapedTaskId = taskId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ const handleRow = await Message.findOne({+ user: userId,+ conversationId,+ content: {+ $elemMatch: {+ type: 'tool_call',+ 'tool_call.output': new RegExp(`"background_task_id"\\s*:\\s*"${escapedTaskId}"`),+ ...(agentId == null ? {} : agentOwnershipFilter('', agentId)),+ },+ },+ })+ .select({ content: 1 })+ .sort({ createdAt: -1, _id: -1 })+ .lean<IMessage | null>();+ const toolName =+ handleRow == null ? undefined : readBackgroundToolHandle(handleRow, taskId, agentId);+ if (toolName != null) {+ return { status: 'outcome_unknown', toolName };+ }+ }return { status: 'not_found' };}- if (row.unfinished === true) {+ /** Only an owner-process manual poll that already retired automatic+ * delivery may consume a terminal receipt on an unfinished response.+ * Every other claimant waits for finalization, preventing a poll from+ * racing the automatic continuation while the parent generation runs. */+ if (row.unfinished === true && !(kind === 'manual' && allowUnfinished)) {return { status: 'not_ready' };}+ const resolvedMessageId = row.messageId;+ if (typeof resolvedMessageId !== 'string' || resolvedMessageId.length === 0) {+ return { status: 'not_found' };+ }+ const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {};const requestedClaim = readBackgroundToolResultClaim(row, taskId);const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;const candidates: string[] = [];@@ -1512,6 +1627,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messareturn {status: 'claimed',...(requestedClaim == null ? {} : { claim: requestedClaim }),+ ...recoveredSource,};}if (!candidates.includes(taskId)) {@@ -1519,13 +1635,20 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messacandidates.splice(boundedLimit);}const claimedAt = new Date();- const claimStamp = { kind, claimId, claimedAt };+ const claimStamp = {+ kind,+ claimId,+ claimedAt,+ ...(kind === 'manual' && requestedGenerationId != null+ ? { generationId: requestedGenerationId }+ : {}),+ };const updated = await Message.findOneAndUpdate({user: userId,conversationId,- messageId,- unfinished: { $ne: true },+ messageId: resolvedMessageId,+ ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),content: {$elemMatch: {type: 'tool_call',@@ -1617,10 +1740,11 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messaconst results = parseBackgroundToolResults(updated, { kind, claimId });const competingClaim = readBackgroundToolResultClaim(updated, taskId);return results.some((result) => result.taskId === taskId)- ? { status: 'acquired', results }+ ? { status: 'acquired', results, ...recoveredSource }: {status: 'claimed',...(competingClaim == null ? {} : { claim: competingClaim }),+ ...recoveredSource,};}diff --git a/packages/data-schemas/src/types/message.ts b/packages/data-schemas/src/types/message.tsindex 18911f39d3..56c381c91e 100644--- a/packages/data-schemas/src/types/message.ts+++ b/packages/data-schemas/src/types/message.ts@@ -114,6 +114,8 @@ export interface IMessage extends Document {kind: 'manual' | 'wakeup';claimId: string;claimedAt: Date;+ /** Response generation that owns a manual delivery claim. */+ generationId?: string;};controlReceipts?: ISubagentTaskControlReceipt[];};