Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧷 fix: Wait for Long Dispatch Turns Before Giving Up Background Code Results (#16345)

Danny Avilac6282fa888f0
diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js
index 4b27bde7fe..a42afb3625 100644
--- a/api/server/controllers/agents/callbacks.js
+++ b/api/server/controllers/agents/callbacks.js
@@ -20,6 +20,7 @@ const {
sendEvent,
computeUsageCostUSD,
GenerationJobManager,
+ waitForGenerationSettled,
writeAttachmentEvent,
createToolExecuteHandler,
createOwnedToolEndHandler,
@@ -1286,14 +1287,20 @@ function createPtcProgressEmitter({ res, streamId = null, jobCreatedAt }) {
* output?: string;
* attachments?: Object[];
* }) => Promise<boolean>} params.updateToolCallResult
+ * @param {number} [params.jobCreatedAt] - Immutable dispatch generation epoch.
+ * @param {string} [params.streamId] - The stream owning that epoch.
*/
-function createBackgroundCodeResultHandler({ req, updateToolCallResult }) {
+function createBackgroundCodeResultHandler({ req, updateToolCallResult, jobCreatedAt, streamId }) {
return createCodeHarvestHandler({
req,
updateToolCallResult,
preflightCodeOutputBatch,
processCodeOutput,
runPreviewFinalize,
+ generationCreatedAt: jobCreatedAt,
+ generationStreamId: streamId,
+ waitForGenerationSettled: (conversationId, options) =>
+ waitForGenerationSettled(GenerationJobManager, conversationId, options),
});
}
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index 875d085951..f6f0f3ebd0 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -511,6 +511,8 @@ const initializeClientWithProvider = async ({
},
persistBackgroundCodeResult: createBackgroundCodeResultHandler({
req,
+ streamId,
+ jobCreatedAt,
updateToolCallResult: db.updateToolCallResult,
}),
backgroundToolCompletion: {
diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js
index 62b05309f0..b93442123f 100644
--- a/api/server/services/Endpoints/agents/initialize.spec.js
+++ b/api/server/services/Endpoints/agents/initialize.spec.js
@@ -304,6 +304,9 @@ describe('initializeClient — processAgent ACL gate', () => {
jobCreatedAt: 1234,
}),
);
+ expect(
+ require('~/server/controllers/agents/callbacks').createBackgroundCodeResultHandler,
+ ).toHaveBeenCalledWith(expect.objectContaining({ jobCreatedAt: 1234 }));
expect(createToolEndCallback).toHaveBeenCalledWith(
expect.objectContaining({
streamId: 'conv_1',
diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts
index 5418fdeb3f..70dca23967 100644
--- a/packages/api/src/agents/background.spec.ts
+++ b/packages/api/src/agents/background.spec.ts
@@ -1369,6 +1369,54 @@ describe('BackgroundTaskRegistryClass', () => {
expect(registry.get('u1', 'c1', created.task.id)).toBeUndefined();
});
+ it.each(['completed', 'error', 'cancelled', 'blocked'] as const)(
+ 'preserves pending %s results through both TTLs and releases protection afterward',
+ async (status) => {
+ jest.useFakeTimers();
+ try {
+ const registry = new BackgroundTaskRegistryClass();
+ const created = registry.create({
+ userId: 'u',
+ conversationId: 'c',
+ toolCallId: 'call',
+ toolName: 'execute_code',
+ harvestStarted: true,
+ });
+ if ('atCapacity' in created) throw new Error('Unexpected capacity rejection');
+ if (status === 'cancelled') registry.cancel('u', 'c', created.task.id, 'cancelled');
+ else if (status === 'error') registry.fail('u', 'c', created.task.id, 'failed');
+ else
+ registry.complete('u', 'c', created.task.id, { content: 'result', harvestStarted: true });
+ registry.markCompletionPersistencePending('u', 'c', created.task.id);
+ registry.finishHarvest('u', 'c', created.task.id, [{ file_id: 'file' }]);
+ if (status === 'blocked') registry.blockArtifact('u', 'c', created.task.id, 'blocked');
+ registry.claimResult('u', 'c', created.task.id, { kind: 'manual', claimId: 'poll' });
+ await jest.advanceTimersByTimeAsync(7 * 60 * 60_000);
+ expect(registry.get('u', 'c', created.task.id)).toMatchObject({
+ completionPersistencePending: true,
+ });
+ registry.markCompletionPersistenceFinished('u', 'c', created.task.id);
+ expect(
+ registry.get('u', 'c', created.task.id)?.completionPersistencePending,
+ ).toBeUndefined();
+ if (status === 'blocked')
+ expect(registry.get('u', 'c', created.task.id)).toMatchObject({
+ artifactBlocked: true,
+ artifact: undefined,
+ });
+ if (status === 'completed')
+ expect(registry.get('u', 'c', created.task.id)).toMatchObject({
+ result: 'result',
+ resultClaim: { kind: 'manual', claimId: 'poll' },
+ });
+ await jest.advanceTimersByTimeAsync(61 * 60_000);
+ expect(registry.get('u', 'c', created.task.id)).toBeUndefined();
+ } finally {
+ jest.useRealTimers();
+ }
+ },
+ );
+
it('caps concurrent running tasks per conversation', () => {
const registry = new BackgroundTaskRegistryClass();
let atCapacity = false;
diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts
index 2c9b744ee4..4caf96f09a 100644
--- a/packages/api/src/agents/background.ts
+++ b/packages/api/src/agents/background.ts
@@ -834,7 +834,11 @@ export class BackgroundTaskRegistryClass {
private sweepBucketTasks(bucket: TaskBucket, now: number): void {
for (const [taskId, task] of bucket.tasks) {
- if (task.status !== 'running' && now - task.updatedAt > COMPLETED_TASK_TTL_MS) {
+ if (
+ task.status !== 'running' &&
+ task.completionPersistencePending !== true &&
+ now - task.updatedAt > COMPLETED_TASK_TTL_MS
+ ) {
bucket.tasks.delete(taskId);
}
}
@@ -859,7 +863,13 @@ export class BackgroundTaskRegistryClass {
}
this.lastGlobalSweepAt = now;
for (const [bucketKey, bucket] of this.buckets) {
- if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) {
+ if (
+ now - bucket.lastAccess > IDLE_BUCKET_TTL_MS &&
+ bucket.capacityPermits.size === 0 &&
+ ![...bucket.tasks.values()].some(
+ (task) => task.status === 'running' || task.completionPersistencePending === true,
+ )
+ ) {
this.buckets.delete(bucketKey);
continue;
}
@@ -1615,7 +1625,14 @@ export class BackgroundTaskRegistryClass {
}
markCompletionPersistenceFinished(userId: string, conversationId: string, taskId: string): void {
- this.update(userId, conversationId, taskId, { completionPersistencePending: undefined });
+ const bucket = this.buckets.get(this.key(userId, conversationId));
+ const task = bucket?.tasks.get(taskId);
+ if (bucket == null || task == null) return;
+ /** Even a policy-blocked task must release retention protection. This only
+ * clears lifecycle state; the immutable artifact block remains intact. */
+ task.completionPersistencePending = undefined;
+ task.updatedAt = Date.now();
+ bucket.lastAccess = task.updatedAt;
}
markCompletionPersistenceFailed(userId: string, conversationId: string, taskId: string): void {
diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts
index 37ad6535d6..6eac3eedb1 100644
--- a/packages/api/src/agents/handlers.background.spec.ts
+++ b/packages/api/src/agents/handlers.background.spec.ts
@@ -2621,6 +2621,59 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
expect(poll[0].artifact).toBeUndefined();
});
+ it('releases stored files to same-turn polls while the row patch waits for the dispatch turn', async () => {
+ const state: CodeToolState = { calls: 0 };
+ const emitted: unknown[] = [];
+ const handler = createToolExecuteHandler({
+ loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
+ /** Files are stored; the row patch then waits for the long dispatch turn. */
+ persistBackgroundCodeResult: (params) => {
+ params.onFilesPersisted?.([{ file_id: 'f1', toolCallId: params.toolCallId }]);
+ return new Promise(() => undefined);
+ },
+ emitAttachment: (attachment) => {
+ emitted.push(attachment);
+ },
+ });
+ const configurable = buildConfig(['execute_code']);
+
+ const dispatch = await runBatch(handler, {
+ toolCalls: [codeCall({ id: 'call_code_long_turn' })],
+ agentId: 'a',
+ configurable,
+ metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },
+ });
+ await flushMicrotasks();
+ await flushMicrotasks();
+
+ const poll = (await runBatch(handler, {
+ toolCalls: [
+ {
+ id: 'call_poll_long_turn',
+ name: CHECK_BACKGROUND_TASK_NAME,
+ args: { background_task_id: JSON.parse(dispatch[0].content).background_task_id },
+ },
+ ],
+ agentId: 'a',
+ configurable,
+ metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },
+ })) as Array<{ content: string; artifact?: unknown }>;
+
+ const polled = JSON.parse(poll[0].content);
+ expect(polled.status).toBe('completed');
+ expect(polled.note).toContain('attached to the tool call');
+ expect(emitted[0]).toEqual({ file_id: 'f1', toolCallId: 'call_code_long_turn' });
+ expect(poll[0].artifact).toEqual(CODE_ARTIFACT);
+ /** Still protected from retention eviction while the row patch waits. */
+ expect(
+ backgroundTaskRegistry.get(
+ 'exec_user',
+ 'exec_convo_code_long_turn',
+ JSON.parse(dispatch[0].content).background_task_id,
+ )?.completionPersistencePending,
+ ).toBe(true);
+ });
+
it('falls back to poll-turn delivery when the harvest fails (files not lost)', async () => {
const state: CodeToolState = { calls: 0 };
const toolEndCalls: Array<{ name?: string; artifact?: unknown }> = [];
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index edbab15caa..601d2dd077 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -315,6 +315,7 @@ export interface ToolExecuteOptions {
reapply?: boolean;
backgroundTask?: BackgroundToolResultState;
resolveBackgroundTask?: () => BackgroundToolResultState;
+ onFilesPersisted?: (attachments: unknown[]) => void;
}) => Promise<{ attachments?: unknown[]; deliveryReady?: boolean } | null>;
/** Shared ordinary-tool completion lifecycle. The delivery is registered
* before invoke; settlement is persisted onto the original response row. */
@@ -6169,6 +6170,13 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
: { backgroundTask, resolveBackgroundTask }),
output: params.output ?? localTask?.result,
artifact: params.artifact,
+ onFilesPersisted: (attachments) =>
+ backgroundTaskRegistry.finishHarvest(
+ backgroundUserId,
+ backgroundConversationId,
+ task.id,
+ attachments,
+ ),
});
if (persisted == null) {
/** Harvest never persisted anything (missing anchor
@@ -6257,10 +6265,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
artifact?: unknown;
status: 'completed' | 'error' | 'cancelled';
}): Promise<void> => {
- if (harvestEnabled) {
- await persistBackgroundResult(params);
- return;
- }
+ /** Held for the whole persist, including a code harvest that waits for
+ * a long dispatch turn, so retention pressure cannot evict the task. */
backgroundTaskRegistry.markCompletionPersistencePending(
backgroundUserId,
backgroundConversationId,
diff --git a/packages/api/src/agents/harvest.lifecycle.spec.ts b/packages/api/src/agents/harvest.lifecycle.spec.ts
new file mode 100644
index 0000000000..a41569f644
--- /dev/null
+++ b/packages/api/src/agents/harvest.lifecycle.spec.ts
@@ -0,0 +1,156 @@
+import type { GenerationSettlementState } from '../stream/interfaces/IJobStore';
+import type { GenerationSettledListener } from '../stream/GenerationJobManager';
+import type { GenerationSettledSource } from '../stream/settled';
+import type { GenerationJobStatus } from '../types/stream';
+import type { ServerRequest } from '~/types';
+import { InMemoryEventTransport } from '../stream/implementations/InMemoryEventTransport';
+import { InMemoryJobStore } from '../stream/implementations/InMemoryJobStore';
+import { GenerationJobManagerClass } from '../stream/GenerationJobManager';
+import { createBackgroundCodeResultHandler } from './harvest';
+import { waitForGenerationSettled } from '../stream/settled';
+
+function sourceForRunningTurn() {
+ const listeners = new Set<GenerationSettledListener>();
+ const job: {
+ status: GenerationJobStatus;
+ createdAt: number;
+ terminalPersistencePending?: boolean;
+ } = { status: 'running', createdAt: 1 };
+ return {
+ job,
+ listeners,
+ getGenerationSettlementState: jest.fn(
+ async (): Promise<GenerationSettlementState | undefined> => ({ ...job }),
+ ),
+ onGenerationSettled: (listener: GenerationSettledListener) => {
+ listeners.add(listener);
+ return () => {
+ listeners.delete(listener);
+ };
+ },
+ settle: () => {
+ job.status = 'complete';
+ for (const listener of [...listeners])
+ listener({
+ streamId: 'conversation',
+ conversationId: 'conversation',
+ userId: 'user',
+ status: 'complete',
+ });
+ },
+ };
+}
+
+function harvest(source: GenerationSettledSource, row: { unfinished: boolean }) {
+ const update = jest.fn(async () => ({ matched: true, unfinished: row.unfinished }));
+ const handler = createBackgroundCodeResultHandler({
+ req: { user: { id: 'user' } } as ServerRequest,
+ preflightCodeOutputBatch: async () => [],
+ processCodeOutput: jest.fn(),
+ updateToolCallResult: update,
+ runPreviewFinalize: jest.fn(),
+ waitForGenerationSettled: (conversationId, options) =>
+ waitForGenerationSettled(source, conversationId, options),
+ });
+ let result: boolean | undefined;
+ const pending = handler({
+ toolName: 'execute_code',
+ toolCallId: 'call',
+ stepId: 'step',
+ messageId: 'message',
+ conversationId: 'conversation',
+ output: 'finished code',
+ backgroundTask: {
+ taskId: 'task',
+ toolName: 'execute_code',
+ status: 'completed',
+ settledAt: new Date(),
+ completionWakeup: true,
+ },
+ }).then((value) => {
+ result = value?.deliveryReady;
+ });
+ return { update, pending, getResult: () => result };
+}
+
+describe('background code harvest generation lifecycle', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ });
+ afterEach(() => {
+ jest.clearAllTimers();
+ jest.useRealTimers();
+ });
+
+ it('keeps harvesting a live turn after a transient first status-read failure', async () => {
+ const source = sourceForRunningTurn();
+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('transient Redis outage'));
+ const row = { unfinished: true };
+ const running = harvest(source, row);
+ await jest.advanceTimersByTimeAsync(1_100_000);
+ const beforeSettle = running.getResult();
+ expect(source.getGenerationSettlementState.mock.calls.length).toBeGreaterThan(1);
+ row.unfinished = false;
+ source.settle();
+ await jest.advanceTimersByTimeAsync(0);
+ expect(beforeSettle).toBeUndefined();
+ await running.pending;
+ expect(running.getResult()).toBe(true);
+ });
+
+ it('anchors the original result when a remote successor replaces the observed generation', async () => {
+ const source = sourceForRunningTurn();
+ const row = { unfinished: true };
+ const running = harvest(source, row);
+ await jest.advanceTimersByTimeAsync(1_000_000);
+ expect(running.update.mock.calls.length).toBe(14);
+ row.unfinished = false;
+ source.job.createdAt = 2;
+ source.job.status = 'requires_action';
+ await jest.advanceTimersByTimeAsync(120_000);
+ const beforeSuccessorSettles = running.getResult();
+ expect(running.update.mock.calls.length).toBe(15);
+ source.settle();
+ await jest.advanceTimersByTimeAsync(0);
+ await running.pending;
+ expect(beforeSuccessorSettles).toBe(true);
+ });
+
+ it('does not retire a result while the real manager is awaiting terminal message persistence', async () => {
+ const manager = new GenerationJobManagerClass();
+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ let finishSave!: () => void;
+ const save = new Promise<void>((resolve) => {
+ finishSave = resolve;
+ });
+ try {
+ const job = await manager.createJob('conversation', 'user', 'conversation');
+ const row = { unfinished: true };
+ const running = harvest(manager, row);
+ await jest.advanceTimersByTimeAsync(1_000_000);
+ const completing = manager.completeJob('conversation', 'provider error', job.createdAt, {
+ beforeErrorPublication: () => save,
+ });
+ await jest.advanceTimersByTimeAsync(120_000);
+ const beforePersistence = running.getResult();
+ expect((await store.getJob('conversation'))?.terminalPersistencePending).toBe(true);
+ row.unfinished = false;
+ finishSave();
+ await completing;
+ await jest.advanceTimersByTimeAsync(0);
+ expect(beforePersistence).toBeUndefined();
+ await running.pending;
+ expect(running.getResult()).toBe(true);
+ } finally {
+ finishSave();
+ await manager.destroy();
+ }
+ });
+});
diff --git a/packages/api/src/agents/harvest.spec.ts b/packages/api/src/agents/harvest.spec.ts
index fe5c98ae0f..8994cfbb55 100644
--- a/packages/api/src/agents/harvest.spec.ts
+++ b/packages/api/src/agents/harvest.spec.ts
@@ -153,3 +153,218 @@ describe('createBackgroundToolResultHandler claim ownership', () => {
);
});
});
+
+describe('createBackgroundCodeResultHandler long dispatch turns', () => {
+ const RETRY_SCHEDULE_MS = 1_000_000;
+ const backgroundTask = {
+ taskId: 'task-1',
+ toolName: 'execute_code',
+ status: 'completed' as const,
+ settledAt: new Date('2026-09-24T00:00:00Z'),
+ completionWakeup: true as const,
+ };
+
+ beforeEach(() => {
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ function createHandler(
+ updateToolCallResult: jest.Mock,
+ waitForGenerationSettled?: (conversationId: string) => Promise<boolean>,
+ ) {
+ return createBackgroundCodeResultHandler({
+ req,
+ preflightCodeOutputBatch: jest
+ .fn()
+ .mockResolvedValue([{ file: params.artifact.files[0], sessionId: 'storage-session' }]),
+ processCodeOutput: jest.fn().mockResolvedValue({ file: { file_id: 'persisted-file' } }),
+ updateToolCallResult,
+ runPreviewFinalize: jest.fn(),
+ ...(waitForGenerationSettled != null && { waitForGenerationSettled }),
+ });
+ }
+
+ it('waits for a dispatch turn that outlives the retry schedule, then anchors', async () => {
+ let turnRunning = true;
+ const updateToolCallResult = jest.fn(async () => ({
+ matched: true,
+ unfinished: turnRunning,
+ }));
+ let settle: (settled: boolean) => void = () => undefined;
+ const waitForGenerationSettled = jest.fn(
+ () =>
+ new Promise<boolean>((resolve) => {
+ settle = resolve;
+ }),
+ );
+ const onFilesPersisted = jest.fn();
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+
+ const result = handler({ ...params, backgroundTask, onFilesPersisted });
+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
+
+ expect(onFilesPersisted).toHaveBeenCalledWith([
+ { file_id: 'persisted-file', stepId: 'step-1' },
+ ]);
+ expect(onFilesPersisted.mock.invocationCallOrder[0]).toBeLessThan(
+ updateToolCallResult.mock.invocationCallOrder[0],
+ );
+ expect(waitForGenerationSettled).toHaveBeenCalledWith('conversation-1', {
+ signal: expect.any(AbortSignal),
+ });
+ const attemptsWhileRunning = updateToolCallResult.mock.calls.length;
+ await jest.advanceTimersByTimeAsync(60 * 60 * 1_000);
+ expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning);
+
+ turnRunning = false;
+ settle(true);
+ await jest.advanceTimersByTimeAsync(0);
+
+ await expect(result).resolves.toEqual({
+ attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }],
+ deliveryReady: true,
+ });
+ expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning + 1);
+ });
+
+ it('waits for a running turn that has not saved its row yet', async () => {
+ let rowSaved = false;
+ const updateToolCallResult = jest.fn(async () => ({ matched: rowSaved, unfinished: false }));
+ const waitForGenerationSettled = jest.fn(async () => {
+ rowSaved = true;
+ return true;
+ });
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+
+ const result = handler({ ...params, backgroundTask });
+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
+
+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
+ expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);
+ });
+
+ it('gives the result up when no generation is running and the row never appears', async () => {
+ const updateToolCallResult = jest.fn(async () => ({ matched: false, unfinished: false }));
+ const waitForGenerationSettled = jest.fn(async () => false);
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+
+ const result = handler({ ...params, backgroundTask });
+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
+
+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
+ expect(updateToolCallResult).toHaveBeenCalledTimes(14 + 5);
+ });
+
+ it('anchors a final save that lands well after the generation reads as settled', async () => {
+ let settledAt: number | undefined;
+ const updateToolCallResult = jest.fn(async () => ({
+ matched: true,
+ unfinished: settledAt == null || Date.now() - settledAt < 40_000,
+ }));
+ const waitForGenerationSettled = jest.fn(async () => {
+ settledAt = Date.now();
+ return true;
+ });
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+
+ const result = handler({ ...params, backgroundTask });
+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
+
+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
+ });
+
+ it('gives the result up when the turn ends without saving the tool call', async () => {
+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));
+ const waitForGenerationSettled = jest.fn(async () => true);
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+
+ const result = handler({ ...params, backgroundTask });
+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);
+
+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
+ expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);
+ });
+
+ it('anchors as soon as the dispatch turn settles instead of at the next scheduled retry', async () => {
+ let turnRunning = true;
+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: turnRunning }));
+ let settle: (settled: boolean) => void = () => undefined;
+ const waitForGenerationSettled = jest.fn(
+ () =>
+ new Promise<boolean>((resolve) => {
+ settle = resolve;
+ }),
+ );
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+ let finished = false;
+
+ const result = handler({ ...params, backgroundTask }).then((value) => {
+ finished = true;
+ return value;
+ });
+ /** 45 s in, the schedule is inside a 30 s gap (next step near 69 s). */
+ await jest.advanceTimersByTimeAsync(45_000);
+ const attemptsBeforeSettle = updateToolCallResult.mock.calls.length;
+ turnRunning = false;
+ settle(true);
+ await jest.advanceTimersByTimeAsync(0);
+
+ expect(finished).toBe(true);
+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));
+ expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsBeforeSettle + 1);
+ });
+
+ it.each([params.conversationId, 'another-conversation'])(
+ 'pins only the epoch owned by the host stream %s',
+ async (generationStreamId) => {
+ const wait = jest.fn(async () => true);
+ const handler = createBackgroundCodeResultHandler({
+ req,
+ preflightCodeOutputBatch: async () => [],
+ processCodeOutput: jest.fn(),
+ runPreviewFinalize: jest.fn(),
+ updateToolCallResult: async () => ({ matched: true, unfinished: false }),
+ generationCreatedAt: 1234,
+ generationStreamId,
+ waitForGenerationSettled: wait,
+ });
+ await handler({ ...params, backgroundTask });
+ expect(wait).toHaveBeenCalledWith(params.conversationId, {
+ ...(generationStreamId === params.conversationId && { generationCreatedAt: 1234 }),
+ signal: expect.any(AbortSignal),
+ });
+ },
+ );
+
+ it('stops listening for the turn once the result is anchored', async () => {
+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: false }));
+ let signal: AbortSignal | undefined;
+ const waitForGenerationSettled = jest.fn(
+ (_conversationId: string, options?: { signal?: AbortSignal }) => {
+ signal = options?.signal;
+ return new Promise<boolean>(() => undefined);
+ },
+ );
+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);
+
+ await expect(handler({ ...params, backgroundTask })).resolves.toEqual(
+ expect.objectContaining({ deliveryReady: true }),
+ );
+ expect(signal?.aborted).toBe(true);
+ });
+
+ it('keeps the bounded schedule when no settlement signal is wired', async () => {
+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));
+ const handler = createHandler(updateToolCallResult);
+
+ const result = handler({ ...params, backgroundTask });
+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);
+
+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));
+ expect(updateToolCallResult).toHaveBeenCalledTimes(14);
+ });
+});
diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts
index 84738597ad..63cd918861 100644
--- a/packages/api/src/agents/harvest.ts
+++ b/packages/api/src/agents/harvest.ts
@@ -1,5 +1,6 @@
import { logger } from '@librechat/data-schemas';
import type { PreparedCodeOutputEntry } from '~/files/code/preflight';
+import type { GenerationSettledWaitOptions } from '../stream/settled';
import type { CodeExecutionContext } from './execution';
import type { ServerRequest } from '~/types';
@@ -12,6 +13,9 @@ import type { ServerRequest } from '~/types';
const BACKGROUND_PATCH_RETRY_DELAYS_MS = [
250, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000, 120_000, 180_000, 240_000, 300_000,
];
+/** Bounded final attempts after the dispatch epoch has finished or disappeared.
+ * Live terminal persistence is waited for explicitly, not inferred from this delay. */
+const SETTLED_PATCH_RETRY_DELAYS_MS = [1_000, 5_000, 15_000, 30_000];
interface HarvestFileRef {
id: string;
name: string;
@@ -51,6 +55,10 @@ export interface BackgroundToolResultState {
}
export interface CodeHarvestDeps {
+ /** Generation identity captured by the host at dispatch, not at harvest time. */
+ generationCreatedAt?: number;
+ /** The stream owning that epoch; nested child conversations must observe their own. */
+ generationStreamId?: string | null;
req: ServerRequest;
/** Data-schemas method: idempotent tool-call part patch + attachment append. */
updateToolCallResult: (params: {
@@ -95,6 +103,12 @@ export interface CodeHarvestDeps {
fileId: string;
previewRevision?: number;
}) => void;
+ /** Waits for the dispatch epoch, including terminal persistence. Read outages
+ * remain retryable; replacement ends the wait without following a successor. */
+ waitForGenerationSettled?: (
+ conversationId: string,
+ options?: GenerationSettledWaitOptions,
+ ) => Promise<boolean>;
}
export interface CodeHarvestParams {
@@ -119,31 +133,48 @@ export interface CodeHarvestParams {
/** Re-reads local claim ownership on every retry so a same-generation
* manual poll cannot be overwritten by a later automatic continuation. */
resolveBackgroundTask?: () => BackgroundToolResultState;
+ /** Called once generated files are stored, before the row patch, which can
+ * wait for the dispatch turn to end: a poll in that turn needs them now. */
+ onFilesPersisted?: (attachments: unknown[]) => void;
}
export type CodeHarvestHandler = (
params: CodeHarvestParams,
) => Promise<{ attachments: unknown[]; deliveryReady?: boolean } | null>;
-const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+type BackgroundResultRowParams = {
+ userId: string;
+ messageId: string;
+ conversationId: string;
+ toolCallId: string;
+ stepId?: string;
+ agentId?: string;
+ output?: string;
+ attachments?: unknown[];
+ backgroundTask?: BackgroundToolResultState;
+ resolveBackgroundTask?: () => BackgroundToolResultState;
+};
-async function persistBackgroundToolResultRow(
+/** Resolves `true` once `interrupt` fires, or `false` after `ms`; never both. */
+function sleepUnlessInterrupted(ms: number, interrupt?: Promise<void>): Promise<boolean> {
+ return new Promise((resolve) => {
+ const timer = setTimeout(() => resolve(false), ms);
+ void interrupt?.then(() => {
+ clearTimeout(timer);
+ resolve(true);
+ });
+ });
+}
+
+/** `false` once `interrupt` cut the schedule short, so the caller can retry at once. */
+async function anchorBackgroundToolResultRow(
updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
- params: {
- userId: string;
- messageId: string;
- conversationId: string;
- toolCallId: string;
- stepId?: string;
- agentId?: string;
- output?: string;
- attachments?: unknown[];
- backgroundTask?: BackgroundToolResultState;
- resolveBackgroundTask?: () => BackgroundToolResultState;
- },
+ params: BackgroundResultRowParams,
+ retryDelaysMs: readonly number[],
+ interrupt?: Promise<void>,
): Promise<boolean> {
const { resolveBackgroundTask, ...persistedParams } = params;
- for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) {
+ for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) {
const currentBackgroundTask = resolveBackgroundTask?.() ?? persistedParams.backgroundTask;
const result = await updateToolCallResult({
...persistedParams,
@@ -153,14 +184,74 @@ async function persistBackgroundToolResultRow(
if (result.matched && !result.unfinished) {
return true;
}
- if (attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length) {
+ if (attempt === retryDelaysMs.length) {
break;
}
- await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]);
+ if (await sleepUnlessInterrupted(retryDelaysMs[attempt], interrupt)) {
+ return false;
+ }
}
return false;
}
+/**
+ * Patches the result onto the dispatch turn's row, which is absent or unfinished
+ * while that turn streams. The patch is retried on a schedule, but the turn
+ * settling is what actually makes the row patchable, so a settle observed
+ * mid-schedule retries at once instead of at the next scheduled step. A turn that
+ * outlives the schedule is waited for rather than given up. Only a positive
+ * settle cuts the schedule short: an unknown generation state keeps the schedule.
+ */
+async function persistBackgroundToolResultRow(
+ updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],
+ params: BackgroundResultRowParams,
+ waitForGenerationSettled?: CodeHarvestDeps['waitForGenerationSettled'],
+ generationCreatedAt?: number,
+): Promise<boolean> {
+ if (waitForGenerationSettled == null) {
+ return anchorBackgroundToolResultRow(
+ updateToolCallResult,
+ params,
+ BACKGROUND_PATCH_RETRY_DELAYS_MS,
+ );
+ }
+ const listening = new AbortController();
+ const settlement = waitForGenerationSettled(params.conversationId, {
+ signal: listening.signal,
+ ...(generationCreatedAt == null ? {} : { generationCreatedAt }),
+ }).catch((error: unknown) => {
+ logger.warn(
+ `[background] Failed waiting for the dispatch turn of message ${params.messageId} to settle:`,
+ error,
+ );
+ return false;
+ });
+ const settledTurn = new Promise<void>((resolve) => {
+ void settlement.then((settled) => {
+ if (settled) resolve();
+ });
+ });
+ try {
+ const anchored = await anchorBackgroundToolResultRow(
+ updateToolCallResult,
+ params,
+ BACKGROUND_PATCH_RETRY_DELAYS_MS,
+ settledTurn,
+ );
+ if (anchored) {
+ return true;
+ }
+ await settlement;
+ return anchorBackgroundToolResultRow(
+ updateToolCallResult,
+ params,
+ SETTLED_PATCH_RETRY_DELAYS_MS,
+ );
+ } finally {
+ listening.abort();
+ }
+}
+
/** Persists an ordinary detached tool result without invoking code-artifact processing. */
export function createBackgroundToolResultHandler(
deps: Pick<CodeHarvestDeps, 'req' | 'updateToolCallResult'>,
@@ -218,6 +309,9 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
preflightCodeOutputBatch,
processCodeOutput,
runPreviewFinalize,
+ waitForGenerationSettled,
+ generationCreatedAt,
+ generationStreamId,
} = deps;
return async ({
toolCallId,
@@ -233,6 +327,7 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
reapply,
backgroundTask,
resolveBackgroundTask,
+ onFilesPersisted,
}) => {
const userId = req.user?.id;
if (!userId || !messageId || !conversationId) {
@@ -311,22 +406,30 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa
}
}
- const deliveryReady = await persistBackgroundToolResultRow(updateToolCallResult, {
- userId,
- messageId,
- conversationId,
- toolCallId,
- stepId,
- agentId,
- output,
- attachments,
- ...(backgroundTask != null ? { backgroundTask } : {}),
- ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
- });
+ onFilesPersisted?.(attachments);
+ const deliveryReady = await persistBackgroundToolResultRow(
+ updateToolCallResult,
+ {
+ userId,
+ messageId,
+ conversationId,
+ toolCallId,
+ stepId,
+ agentId,
+ output,
+ attachments,
+ ...(backgroundTask != null ? { backgroundTask } : {}),
+ ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),
+ },
+ waitForGenerationSettled,
+ generationStreamId == null || generationStreamId === conversationId
+ ? generationCreatedAt
+ : undefined,
+ );
if (!deliveryReady) {
logger.warn(
`[background] Could not anchor code result onto message ${messageId} (tool call ${toolCallId}); ` +
- 'the dispatch turn never persisted. Poll delivery still returns the result.',
+ 'the dispatch turn ended without saving that tool call. Poll delivery still returns the result.',
);
}
return { attachments, ...(backgroundTask != null ? { deliveryReady } : {}) };
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index a92318e29f..7a2120965e 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -20,6 +20,7 @@ import type {
import type { StandardGraph } from '@librechat/agents';
import type {
SerializableJobData,
+ GenerationSettlementState,
CreatedJobData,
IEventTransport,
UsageMetadata,
@@ -3772,9 +3773,21 @@ class GenerationJobManagerClass {
await this.jobStore.releaseIdempotencyKey(legacyKey, expectedClaim);
}
- /**
- * Get job status.
- */
+ /** Observes identity and final-save ownership without attaching a runtime or
+ * promoting a slow terminal writer to stale-owner recovery. */
+ async getGenerationSettlementState(
+ streamId: string,
+ ): Promise<GenerationSettlementState | undefined> {
+ const job = await this.jobStore.getJob(streamId);
+ if (job == null) return undefined;
+ return {
+ createdAt: job.createdAt,
+ status: job.status,
+ terminalPersistencePending: job.terminalPersistencePending,
+ };
+ }
+
+ /** Get job status. */
async getJobStatus(streamId: string): Promise<t.GenerationJobStatus | undefined> {
const jobData = await this.jobStore.getJob(streamId);
return jobData?.status as t.GenerationJobStatus | undefined;
diff --git a/packages/api/src/stream/__tests__/settled.spec.ts b/packages/api/src/stream/__tests__/settled.spec.ts
new file mode 100644
index 0000000000..ff760d45dd
--- /dev/null
+++ b/packages/api/src/stream/__tests__/settled.spec.ts
@@ -0,0 +1,276 @@
+import type { GenerationSettledEvent, GenerationSettledListener } from '../GenerationJobManager';
+import type { GenerationJobStatus } from '../../types/stream';
+import { waitForGenerationSettled } from '../settled';
+
+function createSource(initial: Record<string, GenerationJobStatus>) {
+ const statuses = new Map(Object.entries(initial));
+ const listeners = new Set<GenerationSettledListener>();
+ const epochs = new Map<string, number>();
+ const pending = new Set<string>();
+ return {
+ statuses,
+ epochs,
+ pending,
+ listeners,
+ getGenerationSettlementState: jest.fn(async (streamId: string) => {
+ const status = statuses.get(streamId);
+ return status == null
+ ? undefined
+ : {
+ createdAt: epochs.get(streamId) ?? 1,
+ status,
+ terminalPersistencePending: pending.has(streamId),
+ };
+ }),
+ onGenerationSettled: (listener: GenerationSettledListener) => {
+ listeners.add(listener);
+ return () => {
+ listeners.delete(listener);
+ };
+ },
+ settle(event: Omit<GenerationSettledEvent, 'streamId' | 'userId'>) {
+ statuses.set(event.conversationId, event.status as GenerationJobStatus);
+ for (const listener of listeners) {
+ listener({ ...event, streamId: event.conversationId, userId: 'user-1' });
+ }
+ },
+ };
+}
+
+describe('waitForGenerationSettled', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ it.each([
+ ['no generation', {}],
+ ['a completed generation', { 'conversation-1': 'complete' as const }],
+ ])('resolves false at once for %s', async (_label, initial) => {
+ const source = createSource(initial);
+
+ await expect(waitForGenerationSettled(source, 'conversation-1')).resolves.toBe(false);
+ expect(source.listeners.size).toBe(0);
+ });
+
+ it('resolves when this conversation settles, ignoring others', async () => {
+ const source = createSource({ 'conversation-1': 'running', 'conversation-2': 'running' });
+ let resolved: boolean | undefined;
+ void waitForGenerationSettled(source, 'conversation-1').then((settled) => {
+ resolved = settled;
+ });
+ await jest.advanceTimersByTimeAsync(0);
+
+ source.settle({ conversationId: 'conversation-2', status: 'complete' });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(resolved).toBeUndefined();
+
+ source.settle({ conversationId: 'conversation-1', status: 'complete' });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(resolved).toBe(true);
+ expect(source.listeners.size).toBe(0);
+ });
+
+ it('waits through an approval pause', async () => {
+ const source = createSource({ 'conversation-1': 'requires_action' });
+ let resolved: boolean | undefined;
+ void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(
+ (settled) => {
+ resolved = settled;
+ },
+ );
+
+ await jest.advanceTimersByTimeAsync(5_000);
+ expect(resolved).toBeUndefined();
+ });
+
+ it('observes a generation that settles on another replica', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
+ await jest.advanceTimersByTimeAsync(0);
+
+ source.statuses.set('conversation-1', 'complete');
+ await jest.advanceTimersByTimeAsync(1_000);
+
+ await expect(waiting).resolves.toBe(true);
+ expect(source.listeners.size).toBe(0);
+ });
+
+ it('keeps waiting for as long as the generation stays active by default', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ let resolved: boolean | undefined;
+ void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 60 * 60 * 1_000 }).then(
+ (settled) => {
+ resolved = settled;
+ },
+ );
+
+ await jest.advanceTimersByTimeAsync(48 * 60 * 60 * 1_000);
+ expect(resolved).toBeUndefined();
+
+ source.settle({ conversationId: 'conversation-1', status: 'complete' });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(resolved).toBe(true);
+ });
+
+ it('gives up after the maximum wait', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const waiting = waitForGenerationSettled(source, 'conversation-1', {
+ recheckMs: 1_000,
+ maxWaitMs: 10_000,
+ });
+
+ await jest.advanceTimersByTimeAsync(10_000);
+
+ await expect(waiting).resolves.toBe(false);
+ expect(source.listeners.size).toBe(0);
+ });
+
+ it('stops waiting when the caller aborts', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const controller = new AbortController();
+ const waiting = waitForGenerationSettled(source, 'conversation-1', {
+ signal: controller.signal,
+ });
+ await jest.advanceTimersByTimeAsync(0);
+
+ controller.abort();
+
+ await expect(waiting).resolves.toBe(false);
+ expect(source.listeners.size).toBe(0);
+ });
+
+ it('recovers from initial and later read failures without unsubscribing', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('job store unavailable'));
+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
+ await jest.advanceTimersByTimeAsync(1_000);
+ expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);
+ expect(source.listeners.size).toBe(1);
+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('later outage'));
+ await jest.advanceTimersByTimeAsync(1_000);
+ expect(source.listeners.size).toBe(1);
+ source.statuses.set('conversation-1', 'complete');
+ await jest.advanceTimersByTimeAsync(1_000);
+ await expect(waiting).resolves.toBe(true);
+ expect(source.listeners.size).toBe(0);
+ });
+
+ it.each(['running', 'requires_action'] as const)(
+ 'does not follow a %s successor on another replica',
+ async (status) => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
+ await jest.advanceTimersByTimeAsync(0);
+ source.statuses.set('conversation-1', status);
+ source.epochs.set('conversation-1', 2);
+ await jest.advanceTimersByTimeAsync(1_000);
+ await expect(waiting).resolves.toBe(true);
+ expect(source.listeners.size).toBe(0);
+ },
+ );
+
+ it('uses the dispatch epoch even when the first successful read sees its successor', async () => {
+ const source = createSource({ 'conversation-1': 'requires_action' });
+ source.epochs.set('conversation-1', 2);
+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('outage'));
+ const waiting = waitForGenerationSettled(source, 'conversation-1', {
+ generationCreatedAt: 1,
+ recheckMs: 1_000,
+ });
+ await jest.advanceTimersByTimeAsync(1_000);
+ await expect(waiting).resolves.toBe(true);
+ });
+
+ it.each(['complete', 'error', 'aborted'] as const)(
+ 'waits for %s terminal persistence even after a stale settlement event',
+ async (status) => {
+ const source = createSource({ 'conversation-1': status });
+ source.pending.add('conversation-1');
+ let resolved = false;
+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(
+ (value) => {
+ resolved = true;
+ return value;
+ },
+ );
+ await jest.advanceTimersByTimeAsync(120_000);
+ source.settle({ conversationId: 'conversation-1', status });
+ await jest.advanceTimersByTimeAsync(0);
+ expect(resolved).toBe(false);
+ source.pending.delete('conversation-1');
+ await jest.advanceTimersByTimeAsync(1_000);
+ await expect(waiting).resolves.toBe(true);
+ },
+ );
+
+ it('finishes when the observed job disappears', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
+ await jest.advanceTimersByTimeAsync(0);
+ source.statuses.delete('conversation-1');
+ await jest.advanceTimersByTimeAsync(1_000);
+ await expect(waiting).resolves.toBe(true);
+ });
+
+ it('resolves a pre-aborted wait without subscribing or creating timers', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const controller = new AbortController();
+ controller.abort();
+ await expect(
+ waitForGenerationSettled(source, 'conversation-1', { signal: controller.signal }),
+ ).resolves.toBe(false);
+ expect(source.getGenerationSettlementState).not.toHaveBeenCalled();
+ expect(source.listeners.size).toBe(0);
+ expect(jest.getTimerCount()).toBe(0);
+ });
+
+ it('serializes reads and remembers a settlement event during an in-flight read', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ let release!: () => void;
+ const gate = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ source.getGenerationSettlementState.mockImplementationOnce(async () => {
+ await gate;
+ return { createdAt: 1, status: 'running', terminalPersistencePending: false };
+ });
+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });
+ await jest.advanceTimersByTimeAsync(10_000);
+ source.settle({ conversationId: 'conversation-1', status: 'complete' });
+ source.settle({ conversationId: 'conversation-1', status: 'complete' });
+ expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(1);
+ release();
+ await expect(waiting).resolves.toBe(true);
+ expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);
+ expect(jest.getTimerCount()).toBe(0);
+ });
+
+ it('removes abort listeners and ignores reads completing after cancellation', async () => {
+ const source = createSource({ 'conversation-1': 'running' });
+ const controller = new AbortController();
+ const remove = jest.spyOn(controller.signal, 'removeEventListener');
+ let release!: () => void;
+ const gate = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ source.getGenerationSettlementState.mockImplementationOnce(async () => {
+ await gate;
+ return undefined;
+ });
+ const waiting = waitForGenerationSettled(source, 'conversation-1', {
+ signal: controller.signal,
+ maxWaitMs: 500,
+ });
+ controller.abort();
+ await expect(waiting).resolves.toBe(false);
+ release();
+ await jest.advanceTimersByTimeAsync(1_000);
+ expect(remove).toHaveBeenCalledWith('abort', expect.any(Function));
+ expect(source.listeners.size).toBe(0);
+ expect(jest.getTimerCount()).toBe(0);
+ });
+});
diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts
index ef72b19ee5..9c8180e108 100644
--- a/packages/api/src/stream/index.ts
+++ b/packages/api/src/stream/index.ts
@@ -85,3 +85,7 @@ export { RedisJobStore } from './implementations/RedisJobStore';
export { RedisEventTransport } from './implementations/RedisEventTransport';
export { waitForGenerationPersistence } from './persistence';
+export { waitForGenerationSettled } from './settled';
+export type { GenerationSettledSource, GenerationSettledWaitOptions } from './settled';
+
+export type { GenerationSettlementState } from './interfaces/IJobStore';
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index 4507661aeb..2e7d471b36 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -103,9 +103,13 @@ export type JobStatus = 'running' | 'complete' | 'error' | 'aborted' | 'requires
* Missing markers on pre-rollout records are interpreted as protocol v1. */
export type GenerationProtocolVersion = 1 | 2;
-/**
- * Serializable job data - no object references, suitable for Redis/external storage
- */
+/** Read-only durable evidence, without attaching a runtime or recovering a slow save. */
+export type GenerationSettlementState = Pick<
+ SerializableJobData,
+ 'createdAt' | 'status' | 'terminalPersistencePending'
+>;
+
+/** Serializable job data without object references, suitable for external storage. */
export interface SerializableJobData {
streamId: string;
userId: string;
diff --git a/packages/api/src/stream/settled.ts b/packages/api/src/stream/settled.ts
new file mode 100644
index 0000000000..38fc7a092c
--- /dev/null
+++ b/packages/api/src/stream/settled.ts
@@ -0,0 +1,114 @@
+import { logger } from '@librechat/data-schemas';
+import type { GenerationSettledListener } from './GenerationJobManager';
+import type { GenerationSettlementState } from './interfaces/IJobStore';
+
+const DEFAULT_RECHECK_MS = 60_000;
+
+export interface GenerationSettledSource {
+ getGenerationSettlementState: (
+ streamId: string,
+ ) => Promise<GenerationSettlementState | undefined>;
+ onGenerationSettled: (listener: GenerationSettledListener) => () => void;
+}
+
+export interface GenerationSettledWaitOptions {
+ recheckMs?: number;
+ /** Pin the dispatch generation even if it was replaced before the first read. */
+ generationCreatedAt?: number;
+ /** Optional caller deadline. Store errors are not evidence of settlement. */
+ maxWaitMs?: number;
+ /** Stops waiting (resolving `false`) once the caller no longer needs the answer. */
+ signal?: AbortSignal;
+}
+
+/** Waits for the dispatch generation, including terminal persistence. Events request
+ * a fresh read; they cannot settle an unrelated epoch or bypass an unfinished save.
+ * The first successful read pins the epoch if the caller did not supply it. Reads
+ * are single-flight, with a bounded periodic fallback for remote events and outages. */
+export function waitForGenerationSettled(
+ source: GenerationSettledSource,
+ conversationId: string,
+ {
+ recheckMs = DEFAULT_RECHECK_MS,
+ generationCreatedAt,
+ maxWaitMs,
+ signal,
+ }: GenerationSettledWaitOptions = {},
+): Promise<boolean> {
+ if (signal?.aborted) {
+ return Promise.resolve(false);
+ }
+ return new Promise((resolve) => {
+ let done = false;
+ let reading = false;
+ let readRequested = false;
+ let epoch = generationCreatedAt;
+ let observedActive = epoch != null;
+ let unsubscribe: (() => void) | undefined = undefined;
+ let recheck: NodeJS.Timeout | undefined;
+ let deadline: NodeJS.Timeout | undefined;
+ const abort = (): void => finish(false);
+ const finish = (settled: boolean): void => {
+ if (done) return;
+ done = true;
+ unsubscribe?.();
+ signal?.removeEventListener('abort', abort);
+ clearTimeout(recheck);
+ clearTimeout(deadline);
+ resolve(settled);
+ };
+ const readState = async (): Promise<void> => {
+ if (done) return;
+ if (reading) {
+ readRequested = true;
+ return;
+ }
+ reading = true;
+ clearTimeout(recheck);
+ try {
+ const job = await source.getGenerationSettlementState(conversationId);
+ if (done) return;
+ if (job == null || (epoch != null && epoch !== job.createdAt)) {
+ finish(observedActive);
+ return;
+ }
+ epoch = job.createdAt;
+ const active =
+ job.status === 'running' ||
+ job.status === 'requires_action' ||
+ job.terminalPersistencePending === true;
+ if (!active) {
+ finish(observedActive);
+ return;
+ }
+ observedActive = true;
+ } catch (error) {
+ logger.warn(
+ `[GenerationSettled] Failed to read generation state for ${conversationId}:`,
+ error,
+ );
+ } finally {
+ reading = false;
+ if (!done) {
+ if (readRequested) {
+ readRequested = false;
+ void readState();
+ } else {
+ recheck = setTimeout(() => void readState(), recheckMs);
+ recheck.unref?.();
+ }
+ }
+ }
+ };
+ if (maxWaitMs != null) {
+ deadline = setTimeout(() => finish(false), maxWaitMs);
+ deadline.unref?.();
+ }
+ signal?.addEventListener('abort', abort, { once: true });
+ unsubscribe = source.onGenerationSettled((event) => {
+ if (event.conversationId === conversationId) void readState();
+ });
+ if (done) unsubscribe();
+ else void readState();
+ });
+}