Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🗳️ fix: Recover Durable Background Tool Receipts (#15752)

Danny Avilaa353e117bc3c
diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts
index 1be3ed6c71..b50969d24c 100644
--- a/packages/api/src/agents/background.spec.ts
+++ b/packages/api/src/agents/background.spec.ts
@@ -2096,7 +2096,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
);
});
- it('returns a same-generation result through a local claim that persistence can preserve', async () => {
+ it('lets a later poll collect a receipt without inheriting an abandoned local claim', async () => {
const created = backgroundTaskRegistry.create({
userId: 'claim_user',
conversationId: 'claim_convo',
@@ -2119,25 +2119,33 @@ describe('runCheckBackgroundTask (singleton)', () => {
.fn()
.mockResolvedValueOnce({ status: 'not_ready' })
.mockResolvedValueOnce({ status: 'not_ready' })
+ .mockResolvedValueOnce({ status: 'not_ready' })
.mockResolvedValueOnce({ status: 'acquired', results: [] });
const request = {
userId: 'claim_user',
conversationId: 'claim_convo',
args: { background_task_id: created.task.id },
- toolCallId: 'poll-call',
agentId: 'agent_parent_1',
runId: 'poll-run',
+ generationId: 'response-claim',
claimBackgroundToolResult,
};
- const persisting = JSON.parse(await runCheckBackgroundTask(request));
+ const persisting = JSON.parse(
+ await runCheckBackgroundTask({ ...request, toolCallId: 'poll-call-1' }),
+ );
expect(persisting).toMatchObject({ status: 'result_persisting' });
expect(JSON.stringify(persisting)).not.toContain('CLAIMED RESULT');
- const replay = JSON.parse(await runCheckBackgroundTask(request));
- expect(replay).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });
expect(
backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,
- ).toMatchObject({ kind: 'manual' });
+ ).toBeUndefined();
+ const laterPoll = JSON.parse(
+ await runCheckBackgroundTask({ ...request, toolCallId: 'poll-call-2' }),
+ );
+ expect(laterPoll).toMatchObject({ status: 'completed', result: 'CLAIMED RESULT' });
+ expect(
+ backgroundTaskRegistry.get('claim_user', 'claim_convo', created.task.id)?.resultClaim,
+ ).toMatchObject({ kind: 'manual', generationId: 'response-claim' });
expect(retire).toHaveBeenCalledTimes(1);
expect(retire).toHaveBeenCalledWith('completion claimed by same-generation manual poll', {
onlyIfUnclaimed: true,
@@ -2147,10 +2155,68 @@ describe('runCheckBackgroundTask (singleton)', () => {
messageId: 'response-claim',
taskId: created.task.id,
kind: 'manual',
+ generationId: 'response-claim',
+ allowUnfinished: true,
}),
);
});
+ it('reclaims a dead manual delivery after its owning generation is gone', async () => {
+ const claimBackgroundToolResult = jest
+ .fn()
+ .mockResolvedValueOnce({
+ status: 'claimed',
+ messageId: 'response-recovered',
+ claim: {
+ kind: 'manual',
+ claimId: 'abandoned-poll',
+ generationId: 'response-abandoned',
+ },
+ })
+ .mockResolvedValueOnce({
+ status: 'acquired',
+ messageId: 'response-recovered',
+ results: [
+ {
+ taskId: 'task-recovered-manual',
+ toolCallId: 'call-recovered-manual',
+ toolName: 'mutating_tool',
+ status: 'completed',
+ output: 'mutation already completed',
+ },
+ ],
+ });
+ const recoverDeadBackgroundToolClaim = jest.fn(async () => true);
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'recovered_user',
+ conversationId: 'recovered_convo',
+ args: { background_task_id: 'task-recovered-manual' },
+ toolCallId: 'replacement-poll',
+ runId: 'replacement-run',
+ generationId: 'response-replacement',
+ claimBackgroundToolResult,
+ recoverDeadBackgroundToolClaim,
+ }),
+ );
+
+ expect(result).toMatchObject({
+ status: 'completed',
+ background_task_id: 'task-recovered-manual',
+ result: 'mutation already completed',
+ });
+ expect(recoverDeadBackgroundToolClaim).toHaveBeenCalledWith({
+ userId: 'recovered_user',
+ conversationId: 'recovered_convo',
+ messageId: 'response-recovered',
+ claimId: 'abandoned-poll',
+ kind: 'manual',
+ generationId: 'response-abandoned',
+ });
+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(2);
+ });
+
it('delivers a mandatory live-artifact poll without waiting for its dispatch row', async () => {
const created = backgroundTaskRegistry.create({
userId: 'artifact_poll_user',
@@ -2231,6 +2297,68 @@ describe('runCheckBackgroundTask (singleton)', () => {
expect(JSON.stringify(result)).not.toContain('PRIVATE UNTIL CONTINUATION');
});
+ it.each([
+ { status: 'completed' as const, field: 'result', output: 'RECOVERED RESULT' },
+ { status: 'error' as const, field: 'error', output: 'RECOVERED FAILURE' },
+ ])('recovers a durable $status receipt after process-local state is lost', async (terminal) => {
+ const claimBackgroundToolResult = jest.fn(async () => ({
+ status: 'acquired' as const,
+ messageId: 'response-recovered',
+ results: [
+ {
+ taskId: 'task-recovered',
+ toolCallId: 'call-recovered',
+ toolName: 'slow_tool',
+ status: terminal.status,
+ output: terminal.output,
+ },
+ ],
+ }));
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'recovered_user',
+ conversationId: 'recovered_convo',
+ args: { background_task_id: 'task-recovered' },
+ toolCallId: 'recovery-poll',
+ runId: 'recovery-run',
+ claimBackgroundToolResult,
+ }),
+ );
+
+ expect(result).toMatchObject({
+ status: terminal.status,
+ background_task_id: 'task-recovered',
+ [terminal.field]: terminal.output,
+ });
+ expect(claimBackgroundToolResult).toHaveBeenCalledWith(
+ expect.not.objectContaining({ messageId: expect.anything() }),
+ );
+ });
+
+ it('reports a lost executor without implying that a mutating task is safe to retry', async () => {
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'unknown_user',
+ conversationId: 'unknown_convo',
+ args: { background_task_id: 'task-unknown' },
+ toolCallId: 'unknown-poll',
+ runId: 'unknown-run',
+ claimBackgroundToolResult: async () => ({
+ status: 'outcome_unknown',
+ toolName: 'mutating_tool',
+ }),
+ }),
+ );
+
+ expect(result).toMatchObject({
+ status: 'outcome_unknown',
+ background_task_id: 'task-unknown',
+ tool: 'mutating_tool',
+ });
+ expect(result.message).toContain('Do not repeat a mutating operation automatically');
+ });
+
it('recovers a result through its dead batch-owner claim', async () => {
const created = backgroundTaskRegistry.create({
userId: 'dead_claim_user',
@@ -2256,7 +2384,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
status: 'claimed',
claim: { kind: 'wakeup', claimId: 'sibling-batch-root' },
})
- .mockResolvedValueOnce({ status: 'acquired' });
+ .mockResolvedValueOnce({ status: 'acquired', results: [] });
const recoverDeadBackgroundToolClaim = jest.fn(async () => true);
const result = JSON.parse(
@@ -2472,6 +2600,9 @@ describe('runCheckBackgroundTask (singleton)', () => {
it('polls and one-shot claims a detached subagent result', async () => {
const store = new InMemorySubagentTaskStore();
const subagentTasks: SubagentTaskConfig = { store, scopeId: 'owner:parent-thread' };
+ const claimBackgroundToolResult = jest.fn(async () => {
+ throw new Error('message recovery unavailable');
+ });
const started = store.start({
scopeId: subagentTasks.scopeId,
idempotencyKey: 'parent-run:parent-agent:call-1',
@@ -2494,6 +2625,7 @@ describe('runCheckBackgroundTask (singleton)', () => {
conversationId: 'parent-thread',
args: { background_task_id: started.task.taskId },
subagentTasks,
+ claimBackgroundToolResult,
}),
);
expect(first).toEqual(
@@ -2512,10 +2644,69 @@ describe('runCheckBackgroundTask (singleton)', () => {
conversationId: 'parent-thread',
args: { background_task_id: started.task.taskId },
subagentTasks,
+ claimBackgroundToolResult,
}),
);
expect(second).toEqual(expect.objectContaining({ status: 'claimed', result_claimed: true }));
expect(second.result).toBeUndefined();
+ expect(claimBackgroundToolResult).not.toHaveBeenCalled();
+ });
+
+ it('falls through cleanly when neither background store recognizes a poll id', async () => {
+ const store = new InMemorySubagentTaskStore();
+ const claimBackgroundToolResult = jest.fn(async () => ({ status: 'not_found' as const }));
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'owner',
+ conversationId: 'parent-thread',
+ args: { background_task_id: 'unknown-task' },
+ subagentTasks: { store, scopeId: 'owner:parent-thread' },
+ claimBackgroundToolResult,
+ }),
+ );
+
+ expect(result).toEqual({
+ status: 'not_found',
+ background_task_id: 'unknown-task',
+ message: 'No background task with that id exists in this thread.',
+ });
+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(1);
+ });
+
+ it('recovers an ordinary durable result even when the subagent store is unavailable', async () => {
+ const store = Object.assign(new InMemorySubagentTaskStore(), {
+ claimTask: jest.fn().mockRejectedValue(new SubagentTaskOwnerUnavailableError()),
+ });
+ const claimBackgroundToolResult = jest.fn(async () => ({
+ status: 'acquired' as const,
+ results: [
+ {
+ taskId: 'ordinary-task',
+ toolCallId: 'ordinary-call',
+ toolName: 'mutating_tool',
+ status: 'completed' as const,
+ output: 'ordinary result',
+ },
+ ],
+ }));
+
+ const result = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'owner',
+ conversationId: 'parent-thread',
+ args: { background_task_id: 'ordinary-task' },
+ subagentTasks: { store, scopeId: 'owner:parent-thread' },
+ claimBackgroundToolResult,
+ }),
+ );
+
+ expect(result).toMatchObject({
+ status: 'completed',
+ background_task_id: 'ordinary-task',
+ result: 'ordinary result',
+ });
+ expect(claimBackgroundToolResult).toHaveBeenCalledTimes(1);
});
it('tells a wakeup-enabled parent to yield on an unchanged running subagent', async () => {
diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts
index 02192fef93..8a5c50a0b0 100644
--- a/packages/api/src/agents/background.ts
+++ b/packages/api/src/agents/background.ts
@@ -48,6 +48,10 @@ import type {
SubagentTaskControlResult,
SubagentTaskStore,
} from '@librechat/agents';
+import type {
+ BackgroundToolResultClaim,
+ BackgroundToolResultRecord,
+} from '@librechat/data-schemas';
import type { AgentToolOptions } from 'librechat-data-provider';
import type { CapabilityToolNames } from './selection';
import {
@@ -646,6 +650,7 @@ export interface BackgroundTask {
kind: 'manual' | 'wakeup';
claimId: string;
claimedAt: number;
+ generationId?: string;
};
/** The declared tool may return a process-local live artifact. A terminal
* same-generation poll may therefore deliver from the local claim after it
@@ -653,6 +658,8 @@ export interface BackgroundTask {
* finalize and deadlocking that same generation. */
liveArtifactPollRequired?: boolean;
completionWakeup?: boolean;
+ /** The automatic delivery was durably retired before a manual poll took over. */
+ completionWakeupRetired?: boolean;
/** True while the terminal result is being persisted for automatic delivery. */
completionPersistencePending?: boolean;
/** Process-local cancellation handle for the preregistered durable delivery.
@@ -1552,6 +1559,7 @@ export class BackgroundTaskRegistryClass {
const retired = await task.completionWakeupRetire(reason, options);
if (retired) {
task.completionWakeupRetire = undefined;
+ task.completionWakeupRetired = true;
task.updatedAt = Date.now();
}
return retired;
@@ -1561,7 +1569,7 @@ export class BackgroundTaskRegistryClass {
userId: string,
conversationId: string,
taskId: string,
- claim: { kind: 'manual' | 'wakeup'; claimId: string },
+ claim: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string },
): 'acquired' | 'replay' | 'claimed' | 'not_ready' {
const task = this.get(userId, conversationId, taskId);
if (task == null || task.status === 'running') {
@@ -1793,6 +1801,16 @@ function serializeTask(
};
}
+function serializeDurableTask(task: BackgroundToolResultRecord): SerializedBackgroundTask {
+ return {
+ background_task_id: task.taskId,
+ tool: task.toolName,
+ status: task.status,
+ progress: 1,
+ ...(task.status === 'completed' ? { result: task.output } : { error: task.output }),
+ };
+}
+
interface SerializedSubagentTask {
background_task_id: string;
subagent_thread_id?: string;
@@ -1949,19 +1967,20 @@ export async function runCheckBackgroundTask(params: {
/** Scopes that tool-call id, whose provider ids repeat across runs and agents. */
agentId?: string;
runId?: string;
+ /** Stable response-message identity used to fence abandoned manual claims. */
+ generationId?: string;
subagentTasks?: SubagentTaskConfig;
claimBackgroundToolResult?: (params: {
userId: string;
conversationId: string;
- messageId: string;
+ messageId?: string;
taskId: string;
agentId?: string;
kind: 'manual';
claimId: string;
- }) => Promise<
- | { status: 'acquired' | 'not_found' | 'not_ready' }
- | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }
- >;
+ generationId?: string;
+ allowUnfinished?: boolean;
+ }) => Promise<BackgroundToolResultClaim>;
recoverDeadBackgroundToolClaim?: BackgroundToolDeadClaimRecovery;
}): Promise<string> {
const { userId, conversationId } = params;
@@ -2002,13 +2021,19 @@ export async function runCheckBackgroundTask(params: {
agentId: task.agentId,
kind: 'manual' as const,
claimId: invocationId,
+ ...(params.generationId == null ? {} : { generationId: params.generationId }),
};
let durableClaim = await params.claimBackgroundToolResult(durableClaimInput);
if (durableClaim.status === 'claimed') {
let recovered = false;
let recoveryUnavailable = false;
+ const existingClaim = durableClaim.claim;
+ const recoverableClaim =
+ existingClaim?.kind === 'wakeup' ||
+ (existingClaim?.kind === 'manual' && existingClaim.generationId != null);
if (
- durableClaim.claim?.kind === 'wakeup' &&
+ recoverableClaim &&
+ existingClaim != null &&
params.recoverDeadBackgroundToolClaim != null
) {
try {
@@ -2016,7 +2041,13 @@ export async function runCheckBackgroundTask(params: {
userId,
conversationId,
messageId: task.messageId,
- claimId: durableClaim.claim.claimId,
+ claimId: existingClaim.claimId,
+ ...(existingClaim.kind === 'manual'
+ ? {
+ kind: 'manual' as const,
+ generationId: existingClaim.generationId,
+ }
+ : {}),
});
} catch (error) {
recoveryUnavailable = true;
@@ -2062,37 +2093,39 @@ export async function runCheckBackgroundTask(params: {
* ownership. A live resolver lease wins. Once that resolver is
* irreversibly dead-lettered, a dead-only repair reopens the
* process-local poll fallback without stealing live work. */
- let retired = false;
- try {
- retired = await backgroundTaskRegistry.retireCompletionWakeup(
- userId,
- conversationId,
- taskId,
- 'completion claimed by same-generation manual poll',
- { onlyIfUnclaimed: true },
- );
- if (!retired) {
+ let retired = task.completionWakeupRetired === true;
+ if (!retired) {
+ try {
retired = await backgroundTaskRegistry.retireCompletionWakeup(
userId,
conversationId,
taskId,
- 'dead completion recovered by same-generation manual poll',
- { onlyIfDead: true },
+ 'completion claimed by same-generation manual poll',
+ { onlyIfUnclaimed: true },
);
- if (retired) {
- localClaimNeedsNoDurableConfirmation = true;
- backgroundTaskRegistry.markCompletionPersistenceFailed(
+ if (!retired) {
+ retired = await backgroundTaskRegistry.retireCompletionWakeup(
userId,
conversationId,
taskId,
+ 'dead completion recovered by same-generation manual poll',
+ { onlyIfDead: true },
);
+ if (retired) {
+ localClaimNeedsNoDurableConfirmation = true;
+ backgroundTaskRegistry.markCompletionPersistenceFailed(
+ userId,
+ conversationId,
+ taskId,
+ );
+ }
}
+ } catch (error) {
+ logger.warn(
+ `[background] Failed to retire automatic completion for manual claim ${taskId}:`,
+ error,
+ );
}
- } catch (error) {
- logger.warn(
- `[background] Failed to retire automatic completion for manual claim ${taskId}:`,
- error,
- );
}
if (!retired) {
return JSON.stringify({
@@ -2102,28 +2135,32 @@ export async function runCheckBackgroundTask(params: {
'The task is finished and completion ownership is being settled. Retry this poll shortly.',
});
}
- const localClaim = backgroundTaskRegistry.claimResult(
- userId,
- conversationId,
- taskId,
- { kind: 'manual', claimId: invocationId },
- );
- if (localClaim === 'claimed') {
- return JSON.stringify({
- status: 'delivery_scheduled',
- background_task_id: taskId,
- message: 'This result is already assigned to an automatic continuation.',
- });
- }
- if (localClaim === 'not_ready') {
- return JSON.stringify({
- status: 'result_persisting',
- background_task_id: taskId,
- message:
- 'The task is finished and its result is being made durable. Retry this poll shortly.',
- });
- }
if (task.liveArtifactPollRequired === true) {
+ const localClaim = backgroundTaskRegistry.claimResult(
+ userId,
+ conversationId,
+ taskId,
+ {
+ kind: 'manual',
+ claimId: invocationId,
+ ...(params.generationId == null ? {} : { generationId: params.generationId }),
+ },
+ );
+ if (localClaim === 'claimed') {
+ return JSON.stringify({
+ status: 'delivery_scheduled',
+ background_task_id: taskId,
+ message: 'This result is already assigned to an automatic continuation.',
+ });
+ }
+ if (localClaim === 'not_ready') {
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message:
+ 'The task is finished and its result is being made durable. Retry this poll shortly.',
+ });
+ }
/** The poll is executing inside the still-unfinished dispatch
* generation, so waiting for the durable row would require
* that generation to end before it can obey its mandatory
@@ -2134,12 +2171,18 @@ export async function runCheckBackgroundTask(params: {
localClaimNeedsNoDurableConfirmation = true;
}
}
- /** Ordinary polls do not expose the local result until the durable
- * row has copied this manual claim. The owner-process live-artifact
- * exception above cannot wait for its own generation to finalize;
- * its persister re-reads the local claim after finalization. */
+ /** Ordinary polls claim the durable terminal receipt directly.
+ * They never reserve a process-local claim while the receipt is
+ * absent: a later poll has a different provider tool-call id and
+ * could never take over that abandoned reservation. The live-
+ * artifact exception above cannot wait for its own generation. */
if (!localClaimNeedsNoDurableConfirmation) {
- const reconciledClaim = await params.claimBackgroundToolResult(durableClaimInput);
+ const reconciledClaim = await params.claimBackgroundToolResult({
+ ...durableClaimInput,
+ /** Retirement above makes this owner-process takeover safe;
+ * the terminal row may still be a mid-turn partial save. */
+ allowUnfinished: true,
+ });
if (reconciledClaim.status === 'claimed') {
backgroundTaskRegistry.releaseResultClaim(userId, conversationId, taskId, {
kind: 'manual',
@@ -2159,6 +2202,25 @@ export async function runCheckBackgroundTask(params: {
'The task is finished and its result is being made durable. Retry this poll shortly.',
});
}
+ /** The response's final full save can overwrite a claim stamped
+ * on its unfinished partial row. Mirror only a DURABLY acquired
+ * claim so the persistence retry re-applies that ownership after
+ * finalization; never create an unanchored local reservation. */
+ const mirroredClaim = backgroundTaskRegistry.claimResult(
+ userId,
+ conversationId,
+ taskId,
+ {
+ kind: 'manual',
+ claimId: invocationId,
+ ...(params.generationId == null ? {} : { generationId: params.generationId }),
+ },
+ );
+ if (mirroredClaim === 'claimed') {
+ logger.error(
+ `[background] Durable manual claim for ${taskId} conflicts with process-local ownership.`,
+ );
+ }
}
}
}
@@ -2167,20 +2229,145 @@ export async function runCheckBackgroundTask(params: {
}
const subagentTasks = params.subagentTasks;
+ let subagentPollChecked = false;
+ let subagentPollError: unknown;
+ /** Routed subagents have their own durable/cross-replica store. Resolve
+ * them before ordinary-tool Mongo recovery so two fallback reads—or a
+ * message-store outage—cannot delay or mask a healthy subagent result. */
+ if (action === 'poll' && subagentTasks != null) {
+ subagentPollChecked = true;
+ try {
+ const routedStore = routedSubagentStore(subagentTasks.store);
+ const claim =
+ routedStore == null
+ ? subagentTasks.store.claim(subagentTasks.scopeId, taskId)
+ : await routedStore.claimTask(subagentTasks.scopeId, taskId, invocationId);
+ const claimed = serializeSubagentClaim(
+ claim,
+ agentUsesSubagentCompletionWakeups(subagentTasks, params.agentId),
+ );
+ if (claimed != null) {
+ return JSON.stringify(claimed);
+ }
+ } catch (error) {
+ /** This id may still belong to an ordinary background tool whose
+ * durable receipt is healthy. Defer the subagent failure until after
+ * that independent recovery path has had a chance to identify it. */
+ subagentPollError = error;
+ }
+ }
+
+ if (action === 'poll' && params.claimBackgroundToolResult != null) {
+ const durableClaimInput = {
+ userId,
+ conversationId,
+ taskId,
+ agentId: params.agentId,
+ kind: 'manual' as const,
+ claimId: invocationId,
+ ...(params.generationId == null ? {} : { generationId: params.generationId }),
+ };
+ let durableClaim: BackgroundToolResultClaim;
+ try {
+ durableClaim = await params.claimBackgroundToolResult(durableClaimInput);
+ const claimedResult = durableClaim.status === 'claimed' ? durableClaim : undefined;
+ const existingClaim = claimedResult?.claim;
+ const recoverableClaim =
+ existingClaim?.kind === 'wakeup' ||
+ (existingClaim?.kind === 'manual' && existingClaim.generationId != null);
+ if (
+ recoverableClaim &&
+ existingClaim != null &&
+ claimedResult?.messageId != null &&
+ params.recoverDeadBackgroundToolClaim != null
+ ) {
+ const recovered = await params.recoverDeadBackgroundToolClaim({
+ userId,
+ conversationId,
+ messageId: claimedResult.messageId,
+ claimId: existingClaim.claimId,
+ ...(existingClaim.kind === 'manual'
+ ? {
+ kind: 'manual' as const,
+ generationId: existingClaim.generationId,
+ }
+ : {}),
+ });
+ if (recovered) {
+ durableClaim = await params.claimBackgroundToolResult(durableClaimInput);
+ }
+ }
+ } catch (error) {
+ logger.warn(`[background] Failed to recover durable task ${taskId} during polling:`, error);
+ return JSON.stringify({
+ status: 'result_unavailable',
+ background_task_id: taskId,
+ message:
+ 'The durable task receipt is temporarily unavailable. Do not repeat a mutating operation; retry this status check later.',
+ });
+ }
+ if (durableClaim.status === 'acquired') {
+ const durableTask = durableClaim.results.find((result) => result.taskId === taskId);
+ if (durableTask != null) {
+ return JSON.stringify(serializeDurableTask(durableTask));
+ }
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message: 'The task receipt is settling. Retry this poll shortly.',
+ });
+ }
+ if (durableClaim.status === 'claimed') {
+ return JSON.stringify({
+ status: 'delivery_scheduled',
+ background_task_id: taskId,
+ message: 'This result is already assigned to another poll or automatic continuation.',
+ });
+ }
+ if (durableClaim.status === 'outcome_unknown') {
+ return JSON.stringify({
+ status: 'outcome_unknown',
+ background_task_id: taskId,
+ tool: durableClaim.toolName,
+ message:
+ 'The task was launched, but this server cannot confirm a live executor or a durable terminal receipt. Do not repeat a mutating operation automatically; inspect the target system before retrying.',
+ });
+ }
+ if (durableClaim.status === 'not_ready') {
+ return JSON.stringify({
+ status: 'result_persisting',
+ background_task_id: taskId,
+ message:
+ 'The task exists, but its terminal receipt is not ready. Do not repeat a mutating operation; retry this status check later.',
+ });
+ }
+ }
+
if (subagentTasks != null) {
try {
const routedStore = routedSubagentStore(subagentTasks.store);
if (action === 'poll') {
- const claim =
- routedStore == null
- ? subagentTasks.store.claim(subagentTasks.scopeId, taskId)
- : await routedStore.claimTask(subagentTasks.scopeId, taskId, invocationId);
- const claimed = serializeSubagentClaim(
- claim,
- agentUsesSubagentCompletionWakeups(subagentTasks, params.agentId),
- );
- if (claimed != null) {
- return JSON.stringify(claimed);
+ if (!subagentPollChecked) {
+ const claim =
+ routedStore == null
+ ? subagentTasks.store.claim(subagentTasks.scopeId, taskId)
+ : await routedStore.claimTask(subagentTasks.scopeId, taskId, invocationId);
+ const claimed = serializeSubagentClaim(
+ claim,
+ agentUsesSubagentCompletionWakeups(subagentTasks, params.agentId),
+ );
+ if (claimed != null) {
+ return JSON.stringify(claimed);
+ }
+ } else if (subagentPollError != null) {
+ if (subagentPollError instanceof SubagentTaskOwnerUnavailableError) {
+ return JSON.stringify({
+ status: 'unavailable',
+ background_task_id: taskId,
+ message: subagentPollError.message,
+ });
+ }
+ throw subagentPollError;
}
} else {
const command = buildSubagentControlCommand(args, action);
diff --git a/packages/api/src/agents/backgroundCompletion.ts b/packages/api/src/agents/backgroundCompletion.ts
index 914b9d3822..08b84b0b80 100644
--- a/packages/api/src/agents/backgroundCompletion.ts
+++ b/packages/api/src/agents/backgroundCompletion.ts
@@ -46,6 +46,10 @@ export interface BackgroundToolDeadClaimRecoveryInput {
conversationId: string;
messageId: string;
claimId: string;
+ /** Omitted for the legacy automatic-wakeup recovery path. */
+ kind?: 'manual' | 'wakeup';
+ /** Required to prove that a manual claim's owning generation is no longer active. */
+ generationId?: string;
}
export type BackgroundToolDeadClaimRecovery = (
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
index cd79c4ac52..19a4a9560d 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.spec.ts
@@ -220,6 +220,72 @@ describe('background tool completion wakeups', () => {
});
});
+ it('keeps a manual result claim while its owning generation is active', async () => {
+ const retire = jest.fn(async () => true);
+ const release = jest.fn(async () => true);
+ const getGenerationJob = jest.fn(async () => ({
+ status: 'running',
+ metadata: { responseMessageId: 'response-manual-owner' },
+ }));
+ const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ getGenerationJob,
+ fenceGenerationClaim,
+ );
+
+ await expect(
+ recover({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-result',
+ claimId: 'manual-poll',
+ kind: 'manual',
+ generationId: 'response-manual-owner',
+ }),
+ ).resolves.toBe(false);
+
+ expect(getGenerationJob).toHaveBeenCalledWith('conversation-1');
+ expect(release).not.toHaveBeenCalled();
+ expect(retire).not.toHaveBeenCalled();
+ expect(fenceGenerationClaim).not.toHaveBeenCalled();
+ });
+
+ it('releases a manual result claim after its owning generation is gone', async () => {
+ const retire = jest.fn(async () => true);
+ const release = jest.fn(async () => true);
+ const getGenerationJob = jest.fn(async () => null);
+ const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
+ const recover = createBackgroundToolDeadClaimRecovery(
+ retire,
+ release,
+ getGenerationJob,
+ fenceGenerationClaim,
+ );
+
+ await expect(
+ recover({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-result',
+ claimId: 'manual-poll',
+ kind: 'manual',
+ generationId: 'response-manual-owner',
+ }),
+ ).resolves.toBe(true);
+
+ expect(release).toHaveBeenCalledWith({
+ userId: 'user-1',
+ conversationId: 'conversation-1',
+ messageId: 'response-result',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ });
+ expect(retire).not.toHaveBeenCalled();
+ expect(fenceGenerationClaim).not.toHaveBeenCalled();
+ });
+
it('does not recover a dead delivery while its admitted generation is still active', async () => {
const retire = jest.fn(async () => true);
const release = jest.fn(async () => true);
diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts
index 814f2d319d..03fddfefeb 100644
--- a/packages/api/src/agents/backgroundCompletionWakeup.ts
+++ b/packages/api/src/agents/backgroundCompletionWakeup.ts
@@ -82,6 +82,7 @@ interface GenerationState {
status?: unknown;
metadata?: {
idempotencyClientRequestId?: unknown;
+ responseMessageId?: unknown;
terminalPersistencePending?: unknown;
};
}
@@ -329,6 +330,12 @@ export function createBackgroundToolCompletionWakeupResolver({
if (claim.status === 'claimed') {
return { status: 'settled' };
}
+ if (claim.status === 'outcome_unknown') {
+ throw executionError('The process-local background tool outcome is unknown.', {
+ code: 'BACKGROUND_TOOL_OUTCOME_UNKNOWN',
+ retryable: false,
+ });
+ }
if (claim.status !== 'acquired') {
let producerLease: AgentTriggerProducerLeaseStatus;
try {
@@ -463,7 +470,25 @@ export function createBackgroundToolDeadClaimRecovery(
claimId: string;
}) => Promise<'fenced' | 'started' | 'unavailable'>,
): BackgroundToolDeadClaimRecovery {
- return async ({ userId, conversationId, messageId, claimId }) => {
+ return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
+ if (kind === 'manual') {
+ if (generationId == null || generationId.length === 0) {
+ return false;
+ }
+ const generation = await getGenerationJob(conversationId);
+ if (generation?.metadata?.responseMessageId === generationId && isParentActive(generation)) {
+ return false;
+ }
+ /** Releasing a dead manual DELIVERY only re-presents an already durable
+ * terminal result. It never retries the completed tool mutation. */
+ return releaseClaims({
+ userId,
+ conversationId,
+ messageId,
+ kind: 'manual',
+ claimId,
+ });
+ }
const claimGenerationIsActive = async (): Promise<boolean> => {
const generation = await getGenerationJob(conversationId);
return (
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index b56f795ae6..0e613bc689 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -2,8 +2,12 @@ import { z } from 'zod';
import { logger } from '@librechat/data-schemas';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
+import {
+ backgroundTaskRegistry,
+ runCheckBackgroundTask,
+ CHECK_BACKGROUND_TASK_NAME,
+} from './background';
import { BACKGROUND_TASK_ABORT_GRACE_MS, BACKGROUND_TASK_TIMEOUT_MS } from './backgroundCompletion';
-import { backgroundTaskRegistry, CHECK_BACKGROUND_TASK_NAME } from './background';
import { ContentFilterError } from '../middleware/contentFilter';
import { createToolExecuteHandler } from './handlers';
@@ -110,7 +114,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
backgroundToolCompletion: {
preregister,
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -173,7 +177,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
retire: jest.fn(async () => true),
})),
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -205,7 +209,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
backgroundToolCompletion: {
preregister,
persist: jest.fn(async () => true),
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -240,7 +244,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
backgroundToolCompletion: {
preregister,
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -272,7 +276,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
persist: jest.fn(async () => false),
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -295,6 +299,79 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(retire).toHaveBeenCalledWith('background tool result was not persisted', undefined);
});
+ it('falls back to the settled local result when a poll retired delivery before persistence failed', async () => {
+ let finishPersistence: ((persisted: boolean) => void) | undefined;
+ const persist = jest.fn(
+ () =>
+ new Promise<boolean>((resolve) => {
+ finishPersistence = resolve;
+ }),
+ );
+ const retire = jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false);
+ const tool = makeSearchTool({ calls: 0 });
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [tool] }),
+ backgroundToolCompletion: {
+ preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
+ persist,
+ claim: jest.fn(async () => ({ status: 'not_ready' as const })),
+ },
+ });
+
+ const [dispatch] = await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call-poll-before-failure',
+ name: tool.name,
+ args: { q: 'settle', run_in_background: true },
+ stepId: 'step-poll-before-failure',
+ },
+ ],
+ agentId: 'agent_parent_1',
+ configurable: buildConfig([tool.name]),
+ metadata: { thread_id: 'exec_convo', run_id: 'response-poll-before-failure' },
+ });
+ await flushMicrotasks();
+ const taskId = JSON.parse(dispatch.content).background_task_id as string;
+
+ const waiting = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'exec_user',
+ conversationId: 'exec_convo',
+ args: { background_task_id: taskId },
+ toolCallId: 'manual-poll-1',
+ runId: 'response-poll-before-failure',
+ claimBackgroundToolResult: async () => ({ status: 'not_ready' }),
+ }),
+ );
+ expect(waiting.status).toBe('result_persisting');
+ expect(
+ backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId)?.resultClaim,
+ ).toBeUndefined();
+
+ finishPersistence?.(false);
+ await flushMicrotasks();
+ await flushMicrotasks();
+ expect(
+ backgroundTaskRegistry.get('exec_user', 'exec_convo', taskId)?.completionPersistenceFailed,
+ ).toBe(true);
+
+ const recovered = JSON.parse(
+ await runCheckBackgroundTask({
+ userId: 'exec_user',
+ conversationId: 'exec_convo',
+ args: { background_task_id: taskId },
+ toolCallId: 'manual-poll-2',
+ runId: 'response-poll-before-failure',
+ claimBackgroundToolResult: async () => ({ status: 'not_ready' }),
+ }),
+ );
+ expect(recovered).toMatchObject({
+ status: 'completed',
+ result: 'RESULT for settle',
+ });
+ });
+
it('keeps durable ownership active when an ambiguous persistence failure cannot retire a lease', async () => {
const tool = makeSearchTool({ calls: 0 });
const retire = jest.fn(async () => false);
@@ -356,7 +433,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
backgroundToolCompletion: {
preregister,
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -404,7 +481,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
retire,
})),
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -450,7 +527,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
retire: jest.fn(async () => true),
})),
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -505,7 +582,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
retire: jest.fn(async () => true),
})),
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -561,7 +638,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
retire,
})),
persist,
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
@@ -1772,7 +1849,7 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
backgroundToolCompletion: {
preregister,
persist: jest.fn(async () => true),
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
const configurable = buildConfig(['execute_code']);
@@ -2168,7 +2245,7 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
persist: jest.fn(async () => true),
- claim: jest.fn(async () => ({ status: 'acquired' as const })),
+ claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
const configurable = buildConfig(['execute_code']);
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index 5dc5a3ae70..8886306f41 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -17,9 +17,9 @@ import type {
StreamEventData,
ToolEndCallback as SdkToolEndCallback,
} from '@librechat/agents';
+import type { BackgroundToolResultClaim, ValidationIssue } from '@librechat/data-schemas';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { CodeEnvRef, PtcToolCallEvent } from 'librechat-data-provider';
-import type { ValidationIssue } from '@librechat/data-schemas';
import type {
WorkspaceEditResult,
WorkspacePreviewEditResult,
@@ -283,15 +283,14 @@ export interface ToolExecuteOptions {
claim: (params: {
userId: string;
conversationId: string;
- messageId: string;
+ messageId?: string;
taskId: string;
agentId?: string;
kind: 'manual';
claimId: string;
- }) => Promise<
- | { status: 'acquired' | 'not_found' | 'not_ready' }
- | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }
- >;
+ generationId?: string;
+ allowUnfinished?: boolean;
+ }) => Promise<BackgroundToolResultClaim>;
recoverDeadClaim?: BackgroundToolDeadClaimRecovery;
};
/** Emits an `attachment` SSE event on the current request's live stream. */
@@ -5519,6 +5518,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
kind: current.resultClaim.kind,
claimId: current.resultClaim.claimId,
claimedAt: new Date(current.resultClaim.claimedAt),
+ ...(current.resultClaim.generationId == null
+ ? {}
+ : { generationId: current.resultClaim.generationId }),
},
}
: {}),
@@ -5553,6 +5555,23 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
certainty === 'ambiguous' ? { onlyIfUnclaimed: true } : undefined,
);
if (!retired) {
+ const current = backgroundTaskRegistry.get(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ /** A prior manual poll may already have durably retired
+ * this exact unclaimed delivery. In that case there is
+ * no automatic consumer left to race the process-local
+ * fallback, even though a second retirement is a no-op. */
+ if (current?.completionWakeupRetired === true) {
+ backgroundTaskRegistry.markCompletionPersistenceFailed(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ );
+ return;
+ }
logger.warn(
`[background] Could not retire failed completion delivery for task ${task.id}.`,
);
@@ -6071,6 +6090,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
toolCallId: tc.id,
agentId,
runId: `${backgroundRunId ?? ''}:${tc.turn ?? ''}`,
+ generationId: backgroundRunId,
subagentTasks,
claimBackgroundToolResult: backgroundToolCompletion?.claim,
recoverDeadBackgroundToolClaim: backgroundToolCompletion?.recoverDeadClaim,
diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts
index 84f67cbfdf..0ebda3db75 100644
--- a/packages/api/src/agents/harvest.ts
+++ b/packages/api/src/agents/harvest.ts
@@ -41,6 +41,7 @@ export interface BackgroundToolResultState {
kind: 'manual' | 'wakeup';
claimId: string;
claimedAt: Date;
+ generationId?: string;
};
}
diff --git a/packages/data-provider/src/types/agents.ts b/packages/data-provider/src/types/agents.ts
index cfa7e101bf..e3b4fc8e23 100644
--- a/packages/data-provider/src/types/agents.ts
+++ b/packages/data-provider/src/types/agents.ts
@@ -98,6 +98,8 @@ export namespace Agents {
kind: 'manual' | 'wakeup';
claimId: string;
claimedAt: Date;
+ /** Response generation that owns a manual delivery claim. */
+ generationId?: string;
};
};
/** The tool call was rejected before execution because its input failed schema validation. */
diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts
index 32b8978cce..c9c0d02f3b 100644
--- a/packages/data-schemas/src/methods/message.spec.ts
+++ b/packages/data-schemas/src/methods/message.spec.ts
@@ -1118,6 +1118,315 @@ describe('Message Operations', () => {
expect(task).not.toHaveProperty('completionWakeup');
});
+ it('lets a same-generation manual poll claim an anchored terminal receipt', async () => {
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ unfinished: true,
+ content: [
+ {
+ type: 'tool_call',
+ agentId: 'agent-a',
+ tool_call: {
+ id: 'call-same-generation',
+ name: 'slow_tool',
+ output: 'settled output',
+ backgroundTask: {
+ version: 1,
+ taskId: 'task-same-generation',
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ },
+ },
+ },
+ ],
+ });
+
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-same-generation',
+ kind: 'wakeup',
+ claimId: 'automatic-delivery',
+ }),
+ ).resolves.toEqual({ status: 'not_ready' });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-same-generation',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ }),
+ ).resolves.toEqual({ status: 'not_ready' });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-same-generation',
+ kind: 'manual',
+ claimId: 'manual-poll',
+ allowUnfinished: true,
+ }),
+ ).resolves.toEqual({
+ status: 'acquired',
+ results: [
+ {
+ taskId: 'task-same-generation',
+ toolCallId: 'call-same-generation',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'settled output',
+ agentId: 'agent-a',
+ },
+ ],
+ });
+ });
+
+ it('restores same-generation claim ownership after the final response save', async () => {
+ const terminalContent = [
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-finalize-race',
+ name: 'mutating_tool',
+ output: 'mutation completed',
+ backgroundTask: {
+ version: 1,
+ taskId: 'task-finalize-race',
+ toolName: 'mutating_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ },
+ },
+ },
+ ];
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ unfinished: true,
+ content: terminalContent,
+ });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-finalize-race',
+ kind: 'manual',
+ claimId: 'manual-owner',
+ generationId: 'response-finalize-owner',
+ allowUnfinished: true,
+ }),
+ ).resolves.toMatchObject({ status: 'acquired' });
+
+ /** The final in-memory response did not observe the concurrent claim and
+ * rewrites the content part without it. The detached persistence retry
+ * must restore the mirrored owner before the receipt can be replayed. */
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ unfinished: false,
+ content: terminalContent,
+ });
+ await updateToolCallResult({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ toolCallId: 'call-finalize-race',
+ output: 'mutation completed',
+ backgroundTask: {
+ taskId: 'task-finalize-race',
+ toolName: 'mutating_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ resultClaim: {
+ kind: 'manual',
+ claimId: 'manual-owner',
+ claimedAt: new Date(),
+ generationId: 'response-finalize-owner',
+ },
+ },
+ });
+
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ messageId: 'msg123',
+ taskId: 'task-finalize-race',
+ kind: 'manual',
+ claimId: 'competing-owner',
+ }),
+ ).resolves.toEqual({
+ status: 'claimed',
+ claim: {
+ kind: 'manual',
+ claimId: 'manual-owner',
+ generationId: 'response-finalize-owner',
+ },
+ });
+ });
+
+ it('recovers an unclaimed terminal receipt by task id after process-local state is lost', async () => {
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [
+ {
+ type: 'tool_call',
+ agentId: 'agent-a',
+ tool_call: {
+ id: 'call-recovered',
+ name: 'slow_tool',
+ output: 'durable result',
+ backgroundTask: {
+ version: 1,
+ taskId: 'task-recovered',
+ toolName: 'slow_tool',
+ status: 'completed',
+ settledAt: new Date(),
+ },
+ },
+ },
+ ],
+ });
+
+ const recovery = {
+ status: 'acquired',
+ messageId: 'msg123',
+ results: [
+ {
+ taskId: 'task-recovered',
+ toolCallId: 'call-recovered',
+ toolName: 'slow_tool',
+ status: 'completed',
+ output: 'durable result',
+ agentId: 'agent-a',
+ },
+ ],
+ };
+ const recover = () =>
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ taskId: 'task-recovered',
+ agentId: 'agent-a',
+ kind: 'manual',
+ claimId: 'recovery-poll',
+ generationId: 'response-recovery-owner',
+ });
+ await expect(recover()).resolves.toEqual(recovery);
+ await expect(recover()).resolves.toEqual(recovery);
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ taskId: 'task-recovered',
+ agentId: 'agent-a',
+ kind: 'manual',
+ claimId: 'competing-poll',
+ }),
+ ).resolves.toEqual({
+ status: 'claimed',
+ messageId: 'msg123',
+ claim: {
+ kind: 'manual',
+ claimId: 'recovery-poll',
+ generationId: 'response-recovery-owner',
+ },
+ });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'another-user',
+ conversationId: mockMessageData.conversationId as string,
+ taskId: 'task-recovered',
+ kind: 'manual',
+ claimId: 'cross-user-poll',
+ }),
+ ).resolves.toEqual({ status: 'not_found' });
+ });
+
+ it('reports an exact durable launch handle as outcome unknown after executor loss', async () => {
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [
+ {
+ type: 'tool_call',
+ agentId: 'agent-a',
+ tool_call: {
+ id: 'call-lost',
+ name: 'mutating_tool',
+ output: JSON.stringify({
+ background_task_id: 'task-lost',
+ tool: 'mutating_tool',
+ status: 'running',
+ message: 'Use check_background_task to poll.',
+ }),
+ },
+ },
+ ],
+ });
+
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ taskId: 'task-lost',
+ agentId: 'agent-a',
+ kind: 'manual',
+ claimId: 'recovery-poll',
+ }),
+ ).resolves.toEqual({ status: 'outcome_unknown', toolName: 'mutating_tool' });
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ taskId: 'task-lost',
+ agentId: 'agent-b',
+ kind: 'manual',
+ claimId: 'wrong-agent-poll',
+ }),
+ ).resolves.toEqual({ status: 'not_found' });
+ });
+
+ it('leaves a durable subagent handle to the routed subagent store', async () => {
+ await saveMessage(mockCtx, {
+ ...mockMessageData,
+ content: [
+ {
+ type: 'tool_call',
+ tool_call: {
+ id: 'call-subagent',
+ name: 'subagent',
+ output: JSON.stringify({
+ background_task_id: 'task-subagent',
+ subagent_thread_id: 'thread-subagent',
+ tool: 'subagent',
+ subagent_type: 'researcher',
+ status: 'running',
+ progress: 0,
+ }),
+ },
+ },
+ ],
+ });
+
+ await expect(
+ claimBackgroundToolResults({
+ userId: 'user123',
+ conversationId: mockMessageData.conversationId as string,
+ taskId: 'task-subagent',
+ kind: 'manual',
+ claimId: 'subagent-poll',
+ }),
+ ).resolves.toEqual({ status: 'not_found' });
+ });
+
it('claims a terminal result whose persisted claim stamp is a stored null', async () => {
/** A full-row save can persist `resultClaim: null`, and the
* subfield-preserving settle write keeps it where the old whole-object
diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts
index b59de98abe..48c9ee3166 100644
--- a/packages/data-schemas/src/methods/message.ts
+++ b/packages/data-schemas/src/methods/message.ts
@@ -466,8 +466,13 @@ export interface BackgroundToolResultRecord {
export type BackgroundToolResultClaim =
| { status: 'not_found' | 'not_ready' }
- | { status: 'claimed'; claim?: { kind: 'manual' | 'wakeup'; claimId: string } }
- | { status: 'acquired'; results: BackgroundToolResultRecord[] };
+ | { status: 'outcome_unknown'; toolName: string }
+ | {
+ status: 'claimed';
+ claim?: { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string };
+ messageId?: string;
+ }
+ | { status: 'acquired'; results: BackgroundToolResultRecord[]; messageId?: string };
export type SubagentThreadViewMessageRecord = Pick<
IMessage,
@@ -648,17 +653,23 @@ export interface MessageMethods {
kind: 'manual' | 'wakeup';
claimId: string;
claimedAt: Date;
+ generationId?: string;
};
};
}): Promise<{ matched: boolean; unfinished: boolean }>;
claimBackgroundToolResults(params: {
userId: string;
conversationId: string;
- messageId: string;
+ /** Optional on recovery polls after the process-local task registry was lost. */
+ messageId?: string;
taskId: string;
agentId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
+ /** Response generation that owns this manual result delivery. */
+ generationId?: string;
+ /** Manual owner-process takeover after automatic delivery was retired. */
+ allowUnfinished?: boolean;
limit?: number;
}): Promise<BackgroundToolResultClaim>;
releaseBackgroundToolResultClaims(params: {
@@ -1139,6 +1150,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
kind: 'manual' | 'wakeup';
claimId: string;
claimedAt: Date;
+ generationId?: string;
};
};
}): Promise<{ matched: boolean; unfinished: boolean }> {
@@ -1334,7 +1346,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
function readBackgroundToolResultClaim(
row: Pick<IMessage, 'content'>,
taskId: string,
- ): { kind: 'manual' | 'wakeup'; claimId: string } | undefined {
+ ): { kind: 'manual' | 'wakeup'; claimId: string; generationId?: string } | undefined {
for (const part of row.content ?? []) {
if (part == null || typeof part !== 'object' || Array.isArray(part)) {
continue;
@@ -1344,7 +1356,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
tool_call?: {
backgroundTask?: {
taskId?: unknown;
- resultClaim?: { kind?: unknown; claimId?: unknown };
+ resultClaim?: { kind?: unknown; claimId?: unknown; generationId?: unknown };
};
};
}
@@ -1358,7 +1370,13 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
typeof claim.claimId === 'string' &&
claim.claimId.length > 0
) {
- return { kind: claim.kind, claimId: claim.claimId };
+ return {
+ kind: claim.kind,
+ claimId: claim.claimId,
+ ...(typeof claim.generationId === 'string' && claim.generationId.length > 0
+ ? { generationId: claim.generationId }
+ : {}),
+ };
}
return;
}
@@ -1417,6 +1435,51 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
return results;
}
+ function readBackgroundToolHandle(
+ message: IMessage,
+ taskId: string,
+ agentId?: string,
+ ): string | undefined {
+ for (const part of message.content ?? []) {
+ if (part == null || typeof part !== 'object' || Array.isArray(part)) {
+ continue;
+ }
+ const record = part as {
+ agentId?: unknown;
+ tool_call?: { agentId?: unknown; output?: unknown };
+ };
+ const partAgentId = record.agentId ?? record.tool_call?.agentId;
+ const sameAgent =
+ agentId == null ||
+ partAgentId == null ||
+ (typeof partAgentId === 'string' && partAgentId === agentId);
+ const output = record.tool_call?.output;
+ if (!sameAgent || typeof output !== 'string' || !output.includes(taskId)) {
+ continue;
+ }
+ try {
+ const handle = JSON.parse(output) as {
+ background_task_id?: unknown;
+ subagent_type?: unknown;
+ tool?: unknown;
+ status?: unknown;
+ };
+ if (
+ handle.background_task_id === taskId &&
+ handle.status === 'running' &&
+ typeof handle.subagent_type !== 'string' &&
+ typeof handle.tool === 'string' &&
+ handle.tool.length > 0
+ ) {
+ return handle.tool;
+ }
+ } catch {
+ continue;
+ }
+ }
+ return;
+ }
+
/** Atomically elects manual polling or one automatic continuation. Wakeups
* also claim a bounded set of already-settled siblings from the same parent
* response, avoiding one paid continuation per concurrently completed tool. */
@@ -1428,39 +1491,91 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
agentId,
kind,
claimId,
+ generationId,
+ allowUnfinished = false,
limit = kind === 'wakeup' ? MAX_BACKGROUND_TOOL_RESULT_BATCH : 1,
}: {
userId: string;
conversationId: string;
- messageId: string;
+ messageId?: string;
taskId: string;
agentId?: string;
kind: 'manual' | 'wakeup';
claimId: string;
+ generationId?: string;
+ allowUnfinished?: boolean;
limit?: number;
}): Promise<BackgroundToolResultClaim> {
+ const requestedMessageId = messageId?.trim();
+ const requestedGenerationId = generationId?.trim();
if (
- messageId.length === 0 ||
- messageId.length > 256 ||
+ (requestedMessageId != null &&
+ (requestedMessageId.length === 0 || requestedMessageId.length > 256)) ||
taskId.length === 0 ||
taskId.length > 256 ||
claimId.length === 0 ||
claimId.length > 128 ||
+ (requestedGenerationId != null &&
+ (requestedGenerationId.length === 0 || requestedGenerationId.length > 256)) ||
+ (requestedGenerationId != null && kind !== 'manual') ||
+ (allowUnfinished && kind !== 'manual') ||
(kind !== 'manual' && kind !== 'wakeup')
) {
throw new TypeError('Invalid background tool result claim');
}
const boundedLimit = Math.max(1, Math.min(MAX_BACKGROUND_TOOL_RESULT_BATCH, limit));
const Message = mongoose.models.Message as Model<IMessage>;
- const row = await Message.findOne({ user: userId, conversationId, messageId })
- .select({ content: 1, unfinished: 1 })
+ const row = await Message.findOne({
+ user: userId,
+ conversationId,
+ ...(requestedMessageId != null
+ ? { messageId: requestedMessageId }
+ : {
+ content: {
+ $elemMatch: { 'tool_call.backgroundTask.taskId': taskId },
+ },
+ }),
+ })
+ .select({ content: 1, unfinished: 1, messageId: 1 })
+ .sort({ createdAt: -1, _id: -1 })
.lean<IMessage | null>();
if (row == null) {
+ if (requestedMessageId == null) {
+ const escapedTaskId = taskId.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ const handleRow = await Message.findOne({
+ user: userId,
+ conversationId,
+ content: {
+ $elemMatch: {
+ type: 'tool_call',
+ 'tool_call.output': new RegExp(`"background_task_id"\\s*:\\s*"${escapedTaskId}"`),
+ ...(agentId == null ? {} : agentOwnershipFilter('', agentId)),
+ },
+ },
+ })
+ .select({ content: 1 })
+ .sort({ createdAt: -1, _id: -1 })
+ .lean<IMessage | null>();
+ const toolName =
+ handleRow == null ? undefined : readBackgroundToolHandle(handleRow, taskId, agentId);
+ if (toolName != null) {
+ return { status: 'outcome_unknown', toolName };
+ }
+ }
return { status: 'not_found' };
}
- if (row.unfinished === true) {
+ /** Only an owner-process manual poll that already retired automatic
+ * delivery may consume a terminal receipt on an unfinished response.
+ * Every other claimant waits for finalization, preventing a poll from
+ * racing the automatic continuation while the parent generation runs. */
+ if (row.unfinished === true && !(kind === 'manual' && allowUnfinished)) {
return { status: 'not_ready' };
}
+ const resolvedMessageId = row.messageId;
+ if (typeof resolvedMessageId !== 'string' || resolvedMessageId.length === 0) {
+ return { status: 'not_found' };
+ }
+ const recoveredSource = requestedMessageId == null ? { messageId: resolvedMessageId } : {};
const requestedClaim = readBackgroundToolResultClaim(row, taskId);
const replaying = requestedClaim?.kind === kind && requestedClaim.claimId === claimId;
const candidates: string[] = [];
@@ -1512,6 +1627,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
return {
status: 'claimed',
...(requestedClaim == null ? {} : { claim: requestedClaim }),
+ ...recoveredSource,
};
}
if (!candidates.includes(taskId)) {
@@ -1519,13 +1635,20 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
candidates.splice(boundedLimit);
}
const claimedAt = new Date();
- const claimStamp = { kind, claimId, claimedAt };
+ const claimStamp = {
+ kind,
+ claimId,
+ claimedAt,
+ ...(kind === 'manual' && requestedGenerationId != null
+ ? { generationId: requestedGenerationId }
+ : {}),
+ };
const updated = await Message.findOneAndUpdate(
{
user: userId,
conversationId,
- messageId,
- unfinished: { $ne: true },
+ messageId: resolvedMessageId,
+ ...(kind === 'manual' && allowUnfinished ? {} : { unfinished: { $ne: true } }),
content: {
$elemMatch: {
type: 'tool_call',
@@ -1617,10 +1740,11 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
const results = parseBackgroundToolResults(updated, { kind, claimId });
const competingClaim = readBackgroundToolResultClaim(updated, taskId);
return results.some((result) => result.taskId === taskId)
- ? { status: 'acquired', results }
+ ? { status: 'acquired', results, ...recoveredSource }
: {
status: 'claimed',
...(competingClaim == null ? {} : { claim: competingClaim }),
+ ...recoveredSource,
};
}
diff --git a/packages/data-schemas/src/types/message.ts b/packages/data-schemas/src/types/message.ts
index 18911f39d3..56c381c91e 100644
--- a/packages/data-schemas/src/types/message.ts
+++ b/packages/data-schemas/src/types/message.ts
@@ -114,6 +114,8 @@ export interface IMessage extends Document {
kind: 'manual' | 'wakeup';
claimId: string;
claimedAt: Date;
+ /** Response generation that owns a manual delivery claim. */
+ generationId?: string;
};
controlReceipts?: ISubagentTaskControlReceipt[];
};