Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧷 fix: Wait for Long Dispatch Turns Before Giving Up Background Code Results (#16345)
Danny Avilac6282fa888f0
diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.jsindex 4b27bde7fe..a42afb3625 100644--- a/api/server/controllers/agents/callbacks.js+++ b/api/server/controllers/agents/callbacks.js@@ -20,6 +20,7 @@ const {sendEvent,computeUsageCostUSD,GenerationJobManager,+ waitForGenerationSettled,writeAttachmentEvent,createToolExecuteHandler,createOwnedToolEndHandler,@@ -1286,14 +1287,20 @@ function createPtcProgressEmitter({ res, streamId = null, jobCreatedAt }) {* output?: string;* attachments?: Object[];* }) => Promise<boolean>} params.updateToolCallResult+ * @param {number} [params.jobCreatedAt] - Immutable dispatch generation epoch.+ * @param {string} [params.streamId] - The stream owning that epoch.*/-function createBackgroundCodeResultHandler({ req, updateToolCallResult }) {+function createBackgroundCodeResultHandler({ req, updateToolCallResult, jobCreatedAt, streamId }) {return createCodeHarvestHandler({req,updateToolCallResult,preflightCodeOutputBatch,processCodeOutput,runPreviewFinalize,+ generationCreatedAt: jobCreatedAt,+ generationStreamId: streamId,+ waitForGenerationSettled: (conversationId, options) =>+ waitForGenerationSettled(GenerationJobManager, conversationId, options),});}diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.jsindex 875d085951..f6f0f3ebd0 100644--- a/api/server/services/Endpoints/agents/initialize.js+++ b/api/server/services/Endpoints/agents/initialize.js@@ -511,6 +511,8 @@ const initializeClientWithProvider = async ({},persistBackgroundCodeResult: createBackgroundCodeResultHandler({req,+ streamId,+ jobCreatedAt,updateToolCallResult: db.updateToolCallResult,}),backgroundToolCompletion: {diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.jsindex 62b05309f0..b93442123f 100644--- a/api/server/services/Endpoints/agents/initialize.spec.js+++ b/api/server/services/Endpoints/agents/initialize.spec.js@@ -304,6 +304,9 @@ describe('initializeClient — processAgent ACL gate', () => {jobCreatedAt: 1234,}),);+ expect(+ require('~/server/controllers/agents/callbacks').createBackgroundCodeResultHandler,+ ).toHaveBeenCalledWith(expect.objectContaining({ jobCreatedAt: 1234 }));expect(createToolEndCallback).toHaveBeenCalledWith(expect.objectContaining({streamId: 'conv_1',diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.tsindex 5418fdeb3f..70dca23967 100644--- a/packages/api/src/agents/background.spec.ts+++ b/packages/api/src/agents/background.spec.ts@@ -1369,6 +1369,54 @@ describe('BackgroundTaskRegistryClass', () => {expect(registry.get('u1', 'c1', created.task.id)).toBeUndefined();});+ it.each(['completed', 'error', 'cancelled', 'blocked'] as const)(+ 'preserves pending %s results through both TTLs and releases protection afterward',+ async (status) => {+ jest.useFakeTimers();+ try {+ const registry = new BackgroundTaskRegistryClass();+ const created = registry.create({+ userId: 'u',+ conversationId: 'c',+ toolCallId: 'call',+ toolName: 'execute_code',+ harvestStarted: true,+ });+ if ('atCapacity' in created) throw new Error('Unexpected capacity rejection');+ if (status === 'cancelled') registry.cancel('u', 'c', created.task.id, 'cancelled');+ else if (status === 'error') registry.fail('u', 'c', created.task.id, 'failed');+ else+ registry.complete('u', 'c', created.task.id, { content: 'result', harvestStarted: true });+ registry.markCompletionPersistencePending('u', 'c', created.task.id);+ registry.finishHarvest('u', 'c', created.task.id, [{ file_id: 'file' }]);+ if (status === 'blocked') registry.blockArtifact('u', 'c', created.task.id, 'blocked');+ registry.claimResult('u', 'c', created.task.id, { kind: 'manual', claimId: 'poll' });+ await jest.advanceTimersByTimeAsync(7 * 60 * 60_000);+ expect(registry.get('u', 'c', created.task.id)).toMatchObject({+ completionPersistencePending: true,+ });+ registry.markCompletionPersistenceFinished('u', 'c', created.task.id);+ expect(+ registry.get('u', 'c', created.task.id)?.completionPersistencePending,+ ).toBeUndefined();+ if (status === 'blocked')+ expect(registry.get('u', 'c', created.task.id)).toMatchObject({+ artifactBlocked: true,+ artifact: undefined,+ });+ if (status === 'completed')+ expect(registry.get('u', 'c', created.task.id)).toMatchObject({+ result: 'result',+ resultClaim: { kind: 'manual', claimId: 'poll' },+ });+ await jest.advanceTimersByTimeAsync(61 * 60_000);+ expect(registry.get('u', 'c', created.task.id)).toBeUndefined();+ } finally {+ jest.useRealTimers();+ }+ },+ );+it('caps concurrent running tasks per conversation', () => {const registry = new BackgroundTaskRegistryClass();let atCapacity = false;diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.tsindex 2c9b744ee4..4caf96f09a 100644--- a/packages/api/src/agents/background.ts+++ b/packages/api/src/agents/background.ts@@ -834,7 +834,11 @@ export class BackgroundTaskRegistryClass {private sweepBucketTasks(bucket: TaskBucket, now: number): void {for (const [taskId, task] of bucket.tasks) {- if (task.status !== 'running' && now - task.updatedAt > COMPLETED_TASK_TTL_MS) {+ if (+ task.status !== 'running' &&+ task.completionPersistencePending !== true &&+ now - task.updatedAt > COMPLETED_TASK_TTL_MS+ ) {bucket.tasks.delete(taskId);}}@@ -859,7 +863,13 @@ export class BackgroundTaskRegistryClass {}this.lastGlobalSweepAt = now;for (const [bucketKey, bucket] of this.buckets) {- if (now - bucket.lastAccess > IDLE_BUCKET_TTL_MS && bucket.capacityPermits.size === 0) {+ if (+ now - bucket.lastAccess > IDLE_BUCKET_TTL_MS &&+ bucket.capacityPermits.size === 0 &&+ ![...bucket.tasks.values()].some(+ (task) => task.status === 'running' || task.completionPersistencePending === true,+ )+ ) {this.buckets.delete(bucketKey);continue;}@@ -1615,7 +1625,14 @@ export class BackgroundTaskRegistryClass {}markCompletionPersistenceFinished(userId: string, conversationId: string, taskId: string): void {- this.update(userId, conversationId, taskId, { completionPersistencePending: undefined });+ const bucket = this.buckets.get(this.key(userId, conversationId));+ const task = bucket?.tasks.get(taskId);+ if (bucket == null || task == null) return;+ /** Even a policy-blocked task must release retention protection. This only+ * clears lifecycle state; the immutable artifact block remains intact. */+ task.completionPersistencePending = undefined;+ task.updatedAt = Date.now();+ bucket.lastAccess = task.updatedAt;}markCompletionPersistenceFailed(userId: string, conversationId: string, taskId: string): void {diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.tsindex 37ad6535d6..6eac3eedb1 100644--- a/packages/api/src/agents/handlers.background.spec.ts+++ b/packages/api/src/agents/handlers.background.spec.ts@@ -2621,6 +2621,59 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {expect(poll[0].artifact).toBeUndefined();});+ it('releases stored files to same-turn polls while the row patch waits for the dispatch turn', async () => {+ const state: CodeToolState = { calls: 0 };+ const emitted: unknown[] = [];+ const handler = createToolExecuteHandler({+ loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),+ /** Files are stored; the row patch then waits for the long dispatch turn. */+ persistBackgroundCodeResult: (params) => {+ params.onFilesPersisted?.([{ file_id: 'f1', toolCallId: params.toolCallId }]);+ return new Promise(() => undefined);+ },+ emitAttachment: (attachment) => {+ emitted.push(attachment);+ },+ });+ const configurable = buildConfig(['execute_code']);++ const dispatch = await runBatch(handler, {+ toolCalls: [codeCall({ id: 'call_code_long_turn' })],+ agentId: 'a',+ configurable,+ metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },+ });+ await flushMicrotasks();+ await flushMicrotasks();++ const poll = (await runBatch(handler, {+ toolCalls: [+ {+ id: 'call_poll_long_turn',+ name: CHECK_BACKGROUND_TASK_NAME,+ args: { background_task_id: JSON.parse(dispatch[0].content).background_task_id },+ },+ ],+ agentId: 'a',+ configurable,+ metadata: { thread_id: 'exec_convo_code_long_turn', run_id: 'msg-long-turn' },+ })) as Array<{ content: string; artifact?: unknown }>;++ const polled = JSON.parse(poll[0].content);+ expect(polled.status).toBe('completed');+ expect(polled.note).toContain('attached to the tool call');+ expect(emitted[0]).toEqual({ file_id: 'f1', toolCallId: 'call_code_long_turn' });+ expect(poll[0].artifact).toEqual(CODE_ARTIFACT);+ /** Still protected from retention eviction while the row patch waits. */+ expect(+ backgroundTaskRegistry.get(+ 'exec_user',+ 'exec_convo_code_long_turn',+ JSON.parse(dispatch[0].content).background_task_id,+ )?.completionPersistencePending,+ ).toBe(true);+ });+it('falls back to poll-turn delivery when the harvest fails (files not lost)', async () => {const state: CodeToolState = { calls: 0 };const toolEndCalls: Array<{ name?: string; artifact?: unknown }> = [];diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.tsindex edbab15caa..601d2dd077 100644--- a/packages/api/src/agents/handlers.ts+++ b/packages/api/src/agents/handlers.ts@@ -315,6 +315,7 @@ export interface ToolExecuteOptions {reapply?: boolean;backgroundTask?: BackgroundToolResultState;resolveBackgroundTask?: () => BackgroundToolResultState;+ onFilesPersisted?: (attachments: unknown[]) => void;}) => Promise<{ attachments?: unknown[]; deliveryReady?: boolean } | null>;/** Shared ordinary-tool completion lifecycle. The delivery is registered* before invoke; settlement is persisted onto the original response row. */@@ -6169,6 +6170,13 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand: { backgroundTask, resolveBackgroundTask }),output: params.output ?? localTask?.result,artifact: params.artifact,+ onFilesPersisted: (attachments) =>+ backgroundTaskRegistry.finishHarvest(+ backgroundUserId,+ backgroundConversationId,+ task.id,+ attachments,+ ),});if (persisted == null) {/** Harvest never persisted anything (missing anchor@@ -6257,10 +6265,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandartifact?: unknown;status: 'completed' | 'error' | 'cancelled';}): Promise<void> => {- if (harvestEnabled) {- await persistBackgroundResult(params);- return;- }+ /** Held for the whole persist, including a code harvest that waits for+ * a long dispatch turn, so retention pressure cannot evict the task. */backgroundTaskRegistry.markCompletionPersistencePending(backgroundUserId,backgroundConversationId,diff --git a/packages/api/src/agents/harvest.lifecycle.spec.ts b/packages/api/src/agents/harvest.lifecycle.spec.tsnew file mode 100644index 0000000000..a41569f644--- /dev/null+++ b/packages/api/src/agents/harvest.lifecycle.spec.ts@@ -0,0 +1,156 @@+import type { GenerationSettlementState } from '../stream/interfaces/IJobStore';+import type { GenerationSettledListener } from '../stream/GenerationJobManager';+import type { GenerationSettledSource } from '../stream/settled';+import type { GenerationJobStatus } from '../types/stream';+import type { ServerRequest } from '~/types';+import { InMemoryEventTransport } from '../stream/implementations/InMemoryEventTransport';+import { InMemoryJobStore } from '../stream/implementations/InMemoryJobStore';+import { GenerationJobManagerClass } from '../stream/GenerationJobManager';+import { createBackgroundCodeResultHandler } from './harvest';+import { waitForGenerationSettled } from '../stream/settled';++function sourceForRunningTurn() {+ const listeners = new Set<GenerationSettledListener>();+ const job: {+ status: GenerationJobStatus;+ createdAt: number;+ terminalPersistencePending?: boolean;+ } = { status: 'running', createdAt: 1 };+ return {+ job,+ listeners,+ getGenerationSettlementState: jest.fn(+ async (): Promise<GenerationSettlementState | undefined> => ({ ...job }),+ ),+ onGenerationSettled: (listener: GenerationSettledListener) => {+ listeners.add(listener);+ return () => {+ listeners.delete(listener);+ };+ },+ settle: () => {+ job.status = 'complete';+ for (const listener of [...listeners])+ listener({+ streamId: 'conversation',+ conversationId: 'conversation',+ userId: 'user',+ status: 'complete',+ });+ },+ };+}++function harvest(source: GenerationSettledSource, row: { unfinished: boolean }) {+ const update = jest.fn(async () => ({ matched: true, unfinished: row.unfinished }));+ const handler = createBackgroundCodeResultHandler({+ req: { user: { id: 'user' } } as ServerRequest,+ preflightCodeOutputBatch: async () => [],+ processCodeOutput: jest.fn(),+ updateToolCallResult: update,+ runPreviewFinalize: jest.fn(),+ waitForGenerationSettled: (conversationId, options) =>+ waitForGenerationSettled(source, conversationId, options),+ });+ let result: boolean | undefined;+ const pending = handler({+ toolName: 'execute_code',+ toolCallId: 'call',+ stepId: 'step',+ messageId: 'message',+ conversationId: 'conversation',+ output: 'finished code',+ backgroundTask: {+ taskId: 'task',+ toolName: 'execute_code',+ status: 'completed',+ settledAt: new Date(),+ completionWakeup: true,+ },+ }).then((value) => {+ result = value?.deliveryReady;+ });+ return { update, pending, getResult: () => result };+}++describe('background code harvest generation lifecycle', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ });+ afterEach(() => {+ jest.clearAllTimers();+ jest.useRealTimers();+ });++ it('keeps harvesting a live turn after a transient first status-read failure', async () => {+ const source = sourceForRunningTurn();+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('transient Redis outage'));+ const row = { unfinished: true };+ const running = harvest(source, row);+ await jest.advanceTimersByTimeAsync(1_100_000);+ const beforeSettle = running.getResult();+ expect(source.getGenerationSettlementState.mock.calls.length).toBeGreaterThan(1);+ row.unfinished = false;+ source.settle();+ await jest.advanceTimersByTimeAsync(0);+ expect(beforeSettle).toBeUndefined();+ await running.pending;+ expect(running.getResult()).toBe(true);+ });++ it('anchors the original result when a remote successor replaces the observed generation', async () => {+ const source = sourceForRunningTurn();+ const row = { unfinished: true };+ const running = harvest(source, row);+ await jest.advanceTimersByTimeAsync(1_000_000);+ expect(running.update.mock.calls.length).toBe(14);+ row.unfinished = false;+ source.job.createdAt = 2;+ source.job.status = 'requires_action';+ await jest.advanceTimersByTimeAsync(120_000);+ const beforeSuccessorSettles = running.getResult();+ expect(running.update.mock.calls.length).toBe(15);+ source.settle();+ await jest.advanceTimersByTimeAsync(0);+ await running.pending;+ expect(beforeSuccessorSettles).toBe(true);+ });++ it('does not retire a result while the real manager is awaiting terminal message persistence', async () => {+ const manager = new GenerationJobManagerClass();+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });+ manager.configure({+ jobStore: store,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ manager.initialize();+ let finishSave!: () => void;+ const save = new Promise<void>((resolve) => {+ finishSave = resolve;+ });+ try {+ const job = await manager.createJob('conversation', 'user', 'conversation');+ const row = { unfinished: true };+ const running = harvest(manager, row);+ await jest.advanceTimersByTimeAsync(1_000_000);+ const completing = manager.completeJob('conversation', 'provider error', job.createdAt, {+ beforeErrorPublication: () => save,+ });+ await jest.advanceTimersByTimeAsync(120_000);+ const beforePersistence = running.getResult();+ expect((await store.getJob('conversation'))?.terminalPersistencePending).toBe(true);+ row.unfinished = false;+ finishSave();+ await completing;+ await jest.advanceTimersByTimeAsync(0);+ expect(beforePersistence).toBeUndefined();+ await running.pending;+ expect(running.getResult()).toBe(true);+ } finally {+ finishSave();+ await manager.destroy();+ }+ });+});diff --git a/packages/api/src/agents/harvest.spec.ts b/packages/api/src/agents/harvest.spec.tsindex fe5c98ae0f..8994cfbb55 100644--- a/packages/api/src/agents/harvest.spec.ts+++ b/packages/api/src/agents/harvest.spec.ts@@ -153,3 +153,218 @@ describe('createBackgroundToolResultHandler claim ownership', () => {);});});++describe('createBackgroundCodeResultHandler long dispatch turns', () => {+ const RETRY_SCHEDULE_MS = 1_000_000;+ const backgroundTask = {+ taskId: 'task-1',+ toolName: 'execute_code',+ status: 'completed' as const,+ settledAt: new Date('2026-09-24T00:00:00Z'),+ completionWakeup: true as const,+ };++ beforeEach(() => {+ jest.useFakeTimers();+ });++ afterEach(() => {+ jest.useRealTimers();+ });++ function createHandler(+ updateToolCallResult: jest.Mock,+ waitForGenerationSettled?: (conversationId: string) => Promise<boolean>,+ ) {+ return createBackgroundCodeResultHandler({+ req,+ preflightCodeOutputBatch: jest+ .fn()+ .mockResolvedValue([{ file: params.artifact.files[0], sessionId: 'storage-session' }]),+ processCodeOutput: jest.fn().mockResolvedValue({ file: { file_id: 'persisted-file' } }),+ updateToolCallResult,+ runPreviewFinalize: jest.fn(),+ ...(waitForGenerationSettled != null && { waitForGenerationSettled }),+ });+ }++ it('waits for a dispatch turn that outlives the retry schedule, then anchors', async () => {+ let turnRunning = true;+ const updateToolCallResult = jest.fn(async () => ({+ matched: true,+ unfinished: turnRunning,+ }));+ let settle: (settled: boolean) => void = () => undefined;+ const waitForGenerationSettled = jest.fn(+ () =>+ new Promise<boolean>((resolve) => {+ settle = resolve;+ }),+ );+ const onFilesPersisted = jest.fn();+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);++ const result = handler({ ...params, backgroundTask, onFilesPersisted });+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);++ expect(onFilesPersisted).toHaveBeenCalledWith([+ { file_id: 'persisted-file', stepId: 'step-1' },+ ]);+ expect(onFilesPersisted.mock.invocationCallOrder[0]).toBeLessThan(+ updateToolCallResult.mock.invocationCallOrder[0],+ );+ expect(waitForGenerationSettled).toHaveBeenCalledWith('conversation-1', {+ signal: expect.any(AbortSignal),+ });+ const attemptsWhileRunning = updateToolCallResult.mock.calls.length;+ await jest.advanceTimersByTimeAsync(60 * 60 * 1_000);+ expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning);++ turnRunning = false;+ settle(true);+ await jest.advanceTimersByTimeAsync(0);++ await expect(result).resolves.toEqual({+ attachments: [{ file_id: 'persisted-file', stepId: 'step-1' }],+ deliveryReady: true,+ });+ expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsWhileRunning + 1);+ });++ it('waits for a running turn that has not saved its row yet', async () => {+ let rowSaved = false;+ const updateToolCallResult = jest.fn(async () => ({ matched: rowSaved, unfinished: false }));+ const waitForGenerationSettled = jest.fn(async () => {+ rowSaved = true;+ return true;+ });+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);++ const result = handler({ ...params, backgroundTask });+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);++ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));+ expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);+ });++ it('gives the result up when no generation is running and the row never appears', async () => {+ const updateToolCallResult = jest.fn(async () => ({ matched: false, unfinished: false }));+ const waitForGenerationSettled = jest.fn(async () => false);+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);++ const result = handler({ ...params, backgroundTask });+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);++ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));+ expect(updateToolCallResult).toHaveBeenCalledTimes(14 + 5);+ });++ it('anchors a final save that lands well after the generation reads as settled', async () => {+ let settledAt: number | undefined;+ const updateToolCallResult = jest.fn(async () => ({+ matched: true,+ unfinished: settledAt == null || Date.now() - settledAt < 40_000,+ }));+ const waitForGenerationSettled = jest.fn(async () => {+ settledAt = Date.now();+ return true;+ });+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);++ const result = handler({ ...params, backgroundTask });+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);++ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));+ });++ it('gives the result up when the turn ends without saving the tool call', async () => {+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));+ const waitForGenerationSettled = jest.fn(async () => true);+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);++ const result = handler({ ...params, backgroundTask });+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS + 60_000);++ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));+ expect(waitForGenerationSettled).toHaveBeenCalledTimes(1);+ });++ it('anchors as soon as the dispatch turn settles instead of at the next scheduled retry', async () => {+ let turnRunning = true;+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: turnRunning }));+ let settle: (settled: boolean) => void = () => undefined;+ const waitForGenerationSettled = jest.fn(+ () =>+ new Promise<boolean>((resolve) => {+ settle = resolve;+ }),+ );+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);+ let finished = false;++ const result = handler({ ...params, backgroundTask }).then((value) => {+ finished = true;+ return value;+ });+ /** 45 s in, the schedule is inside a 30 s gap (next step near 69 s). */+ await jest.advanceTimersByTimeAsync(45_000);+ const attemptsBeforeSettle = updateToolCallResult.mock.calls.length;+ turnRunning = false;+ settle(true);+ await jest.advanceTimersByTimeAsync(0);++ expect(finished).toBe(true);+ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: true }));+ expect(updateToolCallResult).toHaveBeenCalledTimes(attemptsBeforeSettle + 1);+ });++ it.each([params.conversationId, 'another-conversation'])(+ 'pins only the epoch owned by the host stream %s',+ async (generationStreamId) => {+ const wait = jest.fn(async () => true);+ const handler = createBackgroundCodeResultHandler({+ req,+ preflightCodeOutputBatch: async () => [],+ processCodeOutput: jest.fn(),+ runPreviewFinalize: jest.fn(),+ updateToolCallResult: async () => ({ matched: true, unfinished: false }),+ generationCreatedAt: 1234,+ generationStreamId,+ waitForGenerationSettled: wait,+ });+ await handler({ ...params, backgroundTask });+ expect(wait).toHaveBeenCalledWith(params.conversationId, {+ ...(generationStreamId === params.conversationId && { generationCreatedAt: 1234 }),+ signal: expect.any(AbortSignal),+ });+ },+ );++ it('stops listening for the turn once the result is anchored', async () => {+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: false }));+ let signal: AbortSignal | undefined;+ const waitForGenerationSettled = jest.fn(+ (_conversationId: string, options?: { signal?: AbortSignal }) => {+ signal = options?.signal;+ return new Promise<boolean>(() => undefined);+ },+ );+ const handler = createHandler(updateToolCallResult, waitForGenerationSettled);++ await expect(handler({ ...params, backgroundTask })).resolves.toEqual(+ expect.objectContaining({ deliveryReady: true }),+ );+ expect(signal?.aborted).toBe(true);+ });++ it('keeps the bounded schedule when no settlement signal is wired', async () => {+ const updateToolCallResult = jest.fn(async () => ({ matched: true, unfinished: true }));+ const handler = createHandler(updateToolCallResult);++ const result = handler({ ...params, backgroundTask });+ await jest.advanceTimersByTimeAsync(RETRY_SCHEDULE_MS);++ await expect(result).resolves.toEqual(expect.objectContaining({ deliveryReady: false }));+ expect(updateToolCallResult).toHaveBeenCalledTimes(14);+ });+});diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.tsindex 84738597ad..63cd918861 100644--- a/packages/api/src/agents/harvest.ts+++ b/packages/api/src/agents/harvest.ts@@ -1,5 +1,6 @@import { logger } from '@librechat/data-schemas';import type { PreparedCodeOutputEntry } from '~/files/code/preflight';+import type { GenerationSettledWaitOptions } from '../stream/settled';import type { CodeExecutionContext } from './execution';import type { ServerRequest } from '~/types';@@ -12,6 +13,9 @@ import type { ServerRequest } from '~/types';const BACKGROUND_PATCH_RETRY_DELAYS_MS = [250, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000, 120_000, 180_000, 240_000, 300_000,];+/** Bounded final attempts after the dispatch epoch has finished or disappeared.+ * Live terminal persistence is waited for explicitly, not inferred from this delay. */+const SETTLED_PATCH_RETRY_DELAYS_MS = [1_000, 5_000, 15_000, 30_000];interface HarvestFileRef {id: string;name: string;@@ -51,6 +55,10 @@ export interface BackgroundToolResultState {}export interface CodeHarvestDeps {+ /** Generation identity captured by the host at dispatch, not at harvest time. */+ generationCreatedAt?: number;+ /** The stream owning that epoch; nested child conversations must observe their own. */+ generationStreamId?: string | null;req: ServerRequest;/** Data-schemas method: idempotent tool-call part patch + attachment append. */updateToolCallResult: (params: {@@ -95,6 +103,12 @@ export interface CodeHarvestDeps {fileId: string;previewRevision?: number;}) => void;+ /** Waits for the dispatch epoch, including terminal persistence. Read outages+ * remain retryable; replacement ends the wait without following a successor. */+ waitForGenerationSettled?: (+ conversationId: string,+ options?: GenerationSettledWaitOptions,+ ) => Promise<boolean>;}export interface CodeHarvestParams {@@ -119,31 +133,48 @@ export interface CodeHarvestParams {/** Re-reads local claim ownership on every retry so a same-generation* manual poll cannot be overwritten by a later automatic continuation. */resolveBackgroundTask?: () => BackgroundToolResultState;+ /** Called once generated files are stored, before the row patch, which can+ * wait for the dispatch turn to end: a poll in that turn needs them now. */+ onFilesPersisted?: (attachments: unknown[]) => void;}export type CodeHarvestHandler = (params: CodeHarvestParams,) => Promise<{ attachments: unknown[]; deliveryReady?: boolean } | null>;-const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));+type BackgroundResultRowParams = {+ userId: string;+ messageId: string;+ conversationId: string;+ toolCallId: string;+ stepId?: string;+ agentId?: string;+ output?: string;+ attachments?: unknown[];+ backgroundTask?: BackgroundToolResultState;+ resolveBackgroundTask?: () => BackgroundToolResultState;+};-async function persistBackgroundToolResultRow(+/** Resolves `true` once `interrupt` fires, or `false` after `ms`; never both. */+function sleepUnlessInterrupted(ms: number, interrupt?: Promise<void>): Promise<boolean> {+ return new Promise((resolve) => {+ const timer = setTimeout(() => resolve(false), ms);+ void interrupt?.then(() => {+ clearTimeout(timer);+ resolve(true);+ });+ });+}++/** `false` once `interrupt` cut the schedule short, so the caller can retry at once. */+async function anchorBackgroundToolResultRow(updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],- params: {- userId: string;- messageId: string;- conversationId: string;- toolCallId: string;- stepId?: string;- agentId?: string;- output?: string;- attachments?: unknown[];- backgroundTask?: BackgroundToolResultState;- resolveBackgroundTask?: () => BackgroundToolResultState;- },+ params: BackgroundResultRowParams,+ retryDelaysMs: readonly number[],+ interrupt?: Promise<void>,): Promise<boolean> {const { resolveBackgroundTask, ...persistedParams } = params;- for (let attempt = 0; attempt <= BACKGROUND_PATCH_RETRY_DELAYS_MS.length; attempt++) {+ for (let attempt = 0; attempt <= retryDelaysMs.length; attempt++) {const currentBackgroundTask = resolveBackgroundTask?.() ?? persistedParams.backgroundTask;const result = await updateToolCallResult({...persistedParams,@@ -153,14 +184,74 @@ async function persistBackgroundToolResultRow(if (result.matched && !result.unfinished) {return true;}- if (attempt === BACKGROUND_PATCH_RETRY_DELAYS_MS.length) {+ if (attempt === retryDelaysMs.length) {break;}- await sleep(BACKGROUND_PATCH_RETRY_DELAYS_MS[attempt]);+ if (await sleepUnlessInterrupted(retryDelaysMs[attempt], interrupt)) {+ return false;+ }}return false;}+/**+ * Patches the result onto the dispatch turn's row, which is absent or unfinished+ * while that turn streams. The patch is retried on a schedule, but the turn+ * settling is what actually makes the row patchable, so a settle observed+ * mid-schedule retries at once instead of at the next scheduled step. A turn that+ * outlives the schedule is waited for rather than given up. Only a positive+ * settle cuts the schedule short: an unknown generation state keeps the schedule.+ */+async function persistBackgroundToolResultRow(+ updateToolCallResult: CodeHarvestDeps['updateToolCallResult'],+ params: BackgroundResultRowParams,+ waitForGenerationSettled?: CodeHarvestDeps['waitForGenerationSettled'],+ generationCreatedAt?: number,+): Promise<boolean> {+ if (waitForGenerationSettled == null) {+ return anchorBackgroundToolResultRow(+ updateToolCallResult,+ params,+ BACKGROUND_PATCH_RETRY_DELAYS_MS,+ );+ }+ const listening = new AbortController();+ const settlement = waitForGenerationSettled(params.conversationId, {+ signal: listening.signal,+ ...(generationCreatedAt == null ? {} : { generationCreatedAt }),+ }).catch((error: unknown) => {+ logger.warn(+ `[background] Failed waiting for the dispatch turn of message ${params.messageId} to settle:`,+ error,+ );+ return false;+ });+ const settledTurn = new Promise<void>((resolve) => {+ void settlement.then((settled) => {+ if (settled) resolve();+ });+ });+ try {+ const anchored = await anchorBackgroundToolResultRow(+ updateToolCallResult,+ params,+ BACKGROUND_PATCH_RETRY_DELAYS_MS,+ settledTurn,+ );+ if (anchored) {+ return true;+ }+ await settlement;+ return anchorBackgroundToolResultRow(+ updateToolCallResult,+ params,+ SETTLED_PATCH_RETRY_DELAYS_MS,+ );+ } finally {+ listening.abort();+ }+}+/** Persists an ordinary detached tool result without invoking code-artifact processing. */export function createBackgroundToolResultHandler(deps: Pick<CodeHarvestDeps, 'req' | 'updateToolCallResult'>,@@ -218,6 +309,9 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHapreflightCodeOutputBatch,processCodeOutput,runPreviewFinalize,+ waitForGenerationSettled,+ generationCreatedAt,+ generationStreamId,} = deps;return async ({toolCallId,@@ -233,6 +327,7 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHareapply,backgroundTask,resolveBackgroundTask,+ onFilesPersisted,}) => {const userId = req.user?.id;if (!userId || !messageId || !conversationId) {@@ -311,22 +406,30 @@ export function createBackgroundCodeResultHandler(deps: CodeHarvestDeps): CodeHa}}- const deliveryReady = await persistBackgroundToolResultRow(updateToolCallResult, {- userId,- messageId,- conversationId,- toolCallId,- stepId,- agentId,- output,- attachments,- ...(backgroundTask != null ? { backgroundTask } : {}),- ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),- });+ onFilesPersisted?.(attachments);+ const deliveryReady = await persistBackgroundToolResultRow(+ updateToolCallResult,+ {+ userId,+ messageId,+ conversationId,+ toolCallId,+ stepId,+ agentId,+ output,+ attachments,+ ...(backgroundTask != null ? { backgroundTask } : {}),+ ...(resolveBackgroundTask != null ? { resolveBackgroundTask } : {}),+ },+ waitForGenerationSettled,+ generationStreamId == null || generationStreamId === conversationId+ ? generationCreatedAt+ : undefined,+ );if (!deliveryReady) {logger.warn(`[background] Could not anchor code result onto message ${messageId} (tool call ${toolCallId}); ` +- 'the dispatch turn never persisted. Poll delivery still returns the result.',+ 'the dispatch turn ended without saving that tool call. Poll delivery still returns the result.',);}return { attachments, ...(backgroundTask != null ? { deliveryReady } : {}) };diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex a92318e29f..7a2120965e 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -20,6 +20,7 @@ import type {import type { StandardGraph } from '@librechat/agents';import type {SerializableJobData,+ GenerationSettlementState,CreatedJobData,IEventTransport,UsageMetadata,@@ -3772,9 +3773,21 @@ class GenerationJobManagerClass {await this.jobStore.releaseIdempotencyKey(legacyKey, expectedClaim);}- /**- * Get job status.- */+ /** Observes identity and final-save ownership without attaching a runtime or+ * promoting a slow terminal writer to stale-owner recovery. */+ async getGenerationSettlementState(+ streamId: string,+ ): Promise<GenerationSettlementState | undefined> {+ const job = await this.jobStore.getJob(streamId);+ if (job == null) return undefined;+ return {+ createdAt: job.createdAt,+ status: job.status,+ terminalPersistencePending: job.terminalPersistencePending,+ };+ }++ /** Get job status. */async getJobStatus(streamId: string): Promise<t.GenerationJobStatus | undefined> {const jobData = await this.jobStore.getJob(streamId);return jobData?.status as t.GenerationJobStatus | undefined;diff --git a/packages/api/src/stream/__tests__/settled.spec.ts b/packages/api/src/stream/__tests__/settled.spec.tsnew file mode 100644index 0000000000..ff760d45dd--- /dev/null+++ b/packages/api/src/stream/__tests__/settled.spec.ts@@ -0,0 +1,276 @@+import type { GenerationSettledEvent, GenerationSettledListener } from '../GenerationJobManager';+import type { GenerationJobStatus } from '../../types/stream';+import { waitForGenerationSettled } from '../settled';++function createSource(initial: Record<string, GenerationJobStatus>) {+ const statuses = new Map(Object.entries(initial));+ const listeners = new Set<GenerationSettledListener>();+ const epochs = new Map<string, number>();+ const pending = new Set<string>();+ return {+ statuses,+ epochs,+ pending,+ listeners,+ getGenerationSettlementState: jest.fn(async (streamId: string) => {+ const status = statuses.get(streamId);+ return status == null+ ? undefined+ : {+ createdAt: epochs.get(streamId) ?? 1,+ status,+ terminalPersistencePending: pending.has(streamId),+ };+ }),+ onGenerationSettled: (listener: GenerationSettledListener) => {+ listeners.add(listener);+ return () => {+ listeners.delete(listener);+ };+ },+ settle(event: Omit<GenerationSettledEvent, 'streamId' | 'userId'>) {+ statuses.set(event.conversationId, event.status as GenerationJobStatus);+ for (const listener of listeners) {+ listener({ ...event, streamId: event.conversationId, userId: 'user-1' });+ }+ },+ };+}++describe('waitForGenerationSettled', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ });++ afterEach(() => {+ jest.useRealTimers();+ });++ it.each([+ ['no generation', {}],+ ['a completed generation', { 'conversation-1': 'complete' as const }],+ ])('resolves false at once for %s', async (_label, initial) => {+ const source = createSource(initial);++ await expect(waitForGenerationSettled(source, 'conversation-1')).resolves.toBe(false);+ expect(source.listeners.size).toBe(0);+ });++ it('resolves when this conversation settles, ignoring others', async () => {+ const source = createSource({ 'conversation-1': 'running', 'conversation-2': 'running' });+ let resolved: boolean | undefined;+ void waitForGenerationSettled(source, 'conversation-1').then((settled) => {+ resolved = settled;+ });+ await jest.advanceTimersByTimeAsync(0);++ source.settle({ conversationId: 'conversation-2', status: 'complete' });+ await jest.advanceTimersByTimeAsync(0);+ expect(resolved).toBeUndefined();++ source.settle({ conversationId: 'conversation-1', status: 'complete' });+ await jest.advanceTimersByTimeAsync(0);+ expect(resolved).toBe(true);+ expect(source.listeners.size).toBe(0);+ });++ it('waits through an approval pause', async () => {+ const source = createSource({ 'conversation-1': 'requires_action' });+ let resolved: boolean | undefined;+ void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(+ (settled) => {+ resolved = settled;+ },+ );++ await jest.advanceTimersByTimeAsync(5_000);+ expect(resolved).toBeUndefined();+ });++ it('observes a generation that settles on another replica', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });+ await jest.advanceTimersByTimeAsync(0);++ source.statuses.set('conversation-1', 'complete');+ await jest.advanceTimersByTimeAsync(1_000);++ await expect(waiting).resolves.toBe(true);+ expect(source.listeners.size).toBe(0);+ });++ it('keeps waiting for as long as the generation stays active by default', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ let resolved: boolean | undefined;+ void waitForGenerationSettled(source, 'conversation-1', { recheckMs: 60 * 60 * 1_000 }).then(+ (settled) => {+ resolved = settled;+ },+ );++ await jest.advanceTimersByTimeAsync(48 * 60 * 60 * 1_000);+ expect(resolved).toBeUndefined();++ source.settle({ conversationId: 'conversation-1', status: 'complete' });+ await jest.advanceTimersByTimeAsync(0);+ expect(resolved).toBe(true);+ });++ it('gives up after the maximum wait', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ const waiting = waitForGenerationSettled(source, 'conversation-1', {+ recheckMs: 1_000,+ maxWaitMs: 10_000,+ });++ await jest.advanceTimersByTimeAsync(10_000);++ await expect(waiting).resolves.toBe(false);+ expect(source.listeners.size).toBe(0);+ });++ it('stops waiting when the caller aborts', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ const controller = new AbortController();+ const waiting = waitForGenerationSettled(source, 'conversation-1', {+ signal: controller.signal,+ });+ await jest.advanceTimersByTimeAsync(0);++ controller.abort();++ await expect(waiting).resolves.toBe(false);+ expect(source.listeners.size).toBe(0);+ });++ it('recovers from initial and later read failures without unsubscribing', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('job store unavailable'));+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });+ await jest.advanceTimersByTimeAsync(1_000);+ expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);+ expect(source.listeners.size).toBe(1);+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('later outage'));+ await jest.advanceTimersByTimeAsync(1_000);+ expect(source.listeners.size).toBe(1);+ source.statuses.set('conversation-1', 'complete');+ await jest.advanceTimersByTimeAsync(1_000);+ await expect(waiting).resolves.toBe(true);+ expect(source.listeners.size).toBe(0);+ });++ it.each(['running', 'requires_action'] as const)(+ 'does not follow a %s successor on another replica',+ async (status) => {+ const source = createSource({ 'conversation-1': 'running' });+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });+ await jest.advanceTimersByTimeAsync(0);+ source.statuses.set('conversation-1', status);+ source.epochs.set('conversation-1', 2);+ await jest.advanceTimersByTimeAsync(1_000);+ await expect(waiting).resolves.toBe(true);+ expect(source.listeners.size).toBe(0);+ },+ );++ it('uses the dispatch epoch even when the first successful read sees its successor', async () => {+ const source = createSource({ 'conversation-1': 'requires_action' });+ source.epochs.set('conversation-1', 2);+ source.getGenerationSettlementState.mockRejectedValueOnce(new Error('outage'));+ const waiting = waitForGenerationSettled(source, 'conversation-1', {+ generationCreatedAt: 1,+ recheckMs: 1_000,+ });+ await jest.advanceTimersByTimeAsync(1_000);+ await expect(waiting).resolves.toBe(true);+ });++ it.each(['complete', 'error', 'aborted'] as const)(+ 'waits for %s terminal persistence even after a stale settlement event',+ async (status) => {+ const source = createSource({ 'conversation-1': status });+ source.pending.add('conversation-1');+ let resolved = false;+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 }).then(+ (value) => {+ resolved = true;+ return value;+ },+ );+ await jest.advanceTimersByTimeAsync(120_000);+ source.settle({ conversationId: 'conversation-1', status });+ await jest.advanceTimersByTimeAsync(0);+ expect(resolved).toBe(false);+ source.pending.delete('conversation-1');+ await jest.advanceTimersByTimeAsync(1_000);+ await expect(waiting).resolves.toBe(true);+ },+ );++ it('finishes when the observed job disappears', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });+ await jest.advanceTimersByTimeAsync(0);+ source.statuses.delete('conversation-1');+ await jest.advanceTimersByTimeAsync(1_000);+ await expect(waiting).resolves.toBe(true);+ });++ it('resolves a pre-aborted wait without subscribing or creating timers', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ const controller = new AbortController();+ controller.abort();+ await expect(+ waitForGenerationSettled(source, 'conversation-1', { signal: controller.signal }),+ ).resolves.toBe(false);+ expect(source.getGenerationSettlementState).not.toHaveBeenCalled();+ expect(source.listeners.size).toBe(0);+ expect(jest.getTimerCount()).toBe(0);+ });++ it('serializes reads and remembers a settlement event during an in-flight read', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ let release!: () => void;+ const gate = new Promise<void>((resolve) => {+ release = resolve;+ });+ source.getGenerationSettlementState.mockImplementationOnce(async () => {+ await gate;+ return { createdAt: 1, status: 'running', terminalPersistencePending: false };+ });+ const waiting = waitForGenerationSettled(source, 'conversation-1', { recheckMs: 1_000 });+ await jest.advanceTimersByTimeAsync(10_000);+ source.settle({ conversationId: 'conversation-1', status: 'complete' });+ source.settle({ conversationId: 'conversation-1', status: 'complete' });+ expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(1);+ release();+ await expect(waiting).resolves.toBe(true);+ expect(source.getGenerationSettlementState).toHaveBeenCalledTimes(2);+ expect(jest.getTimerCount()).toBe(0);+ });++ it('removes abort listeners and ignores reads completing after cancellation', async () => {+ const source = createSource({ 'conversation-1': 'running' });+ const controller = new AbortController();+ const remove = jest.spyOn(controller.signal, 'removeEventListener');+ let release!: () => void;+ const gate = new Promise<void>((resolve) => {+ release = resolve;+ });+ source.getGenerationSettlementState.mockImplementationOnce(async () => {+ await gate;+ return undefined;+ });+ const waiting = waitForGenerationSettled(source, 'conversation-1', {+ signal: controller.signal,+ maxWaitMs: 500,+ });+ controller.abort();+ await expect(waiting).resolves.toBe(false);+ release();+ await jest.advanceTimersByTimeAsync(1_000);+ expect(remove).toHaveBeenCalledWith('abort', expect.any(Function));+ expect(source.listeners.size).toBe(0);+ expect(jest.getTimerCount()).toBe(0);+ });+});diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.tsindex ef72b19ee5..9c8180e108 100644--- a/packages/api/src/stream/index.ts+++ b/packages/api/src/stream/index.ts@@ -85,3 +85,7 @@ export { RedisJobStore } from './implementations/RedisJobStore';export { RedisEventTransport } from './implementations/RedisEventTransport';export { waitForGenerationPersistence } from './persistence';+export { waitForGenerationSettled } from './settled';+export type { GenerationSettledSource, GenerationSettledWaitOptions } from './settled';++export type { GenerationSettlementState } from './interfaces/IJobStore';diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex 4507661aeb..2e7d471b36 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -103,9 +103,13 @@ export type JobStatus = 'running' | 'complete' | 'error' | 'aborted' | 'requires* Missing markers on pre-rollout records are interpreted as protocol v1. */export type GenerationProtocolVersion = 1 | 2;-/**- * Serializable job data - no object references, suitable for Redis/external storage- */+/** Read-only durable evidence, without attaching a runtime or recovering a slow save. */+export type GenerationSettlementState = Pick<+ SerializableJobData,+ 'createdAt' | 'status' | 'terminalPersistencePending'+>;++/** Serializable job data without object references, suitable for external storage. */export interface SerializableJobData {streamId: string;userId: string;diff --git a/packages/api/src/stream/settled.ts b/packages/api/src/stream/settled.tsnew file mode 100644index 0000000000..38fc7a092c--- /dev/null+++ b/packages/api/src/stream/settled.ts@@ -0,0 +1,114 @@+import { logger } from '@librechat/data-schemas';+import type { GenerationSettledListener } from './GenerationJobManager';+import type { GenerationSettlementState } from './interfaces/IJobStore';++const DEFAULT_RECHECK_MS = 60_000;++export interface GenerationSettledSource {+ getGenerationSettlementState: (+ streamId: string,+ ) => Promise<GenerationSettlementState | undefined>;+ onGenerationSettled: (listener: GenerationSettledListener) => () => void;+}++export interface GenerationSettledWaitOptions {+ recheckMs?: number;+ /** Pin the dispatch generation even if it was replaced before the first read. */+ generationCreatedAt?: number;+ /** Optional caller deadline. Store errors are not evidence of settlement. */+ maxWaitMs?: number;+ /** Stops waiting (resolving `false`) once the caller no longer needs the answer. */+ signal?: AbortSignal;+}++/** Waits for the dispatch generation, including terminal persistence. Events request+ * a fresh read; they cannot settle an unrelated epoch or bypass an unfinished save.+ * The first successful read pins the epoch if the caller did not supply it. Reads+ * are single-flight, with a bounded periodic fallback for remote events and outages. */+export function waitForGenerationSettled(+ source: GenerationSettledSource,+ conversationId: string,+ {+ recheckMs = DEFAULT_RECHECK_MS,+ generationCreatedAt,+ maxWaitMs,+ signal,+ }: GenerationSettledWaitOptions = {},+): Promise<boolean> {+ if (signal?.aborted) {+ return Promise.resolve(false);+ }+ return new Promise((resolve) => {+ let done = false;+ let reading = false;+ let readRequested = false;+ let epoch = generationCreatedAt;+ let observedActive = epoch != null;+ let unsubscribe: (() => void) | undefined = undefined;+ let recheck: NodeJS.Timeout | undefined;+ let deadline: NodeJS.Timeout | undefined;+ const abort = (): void => finish(false);+ const finish = (settled: boolean): void => {+ if (done) return;+ done = true;+ unsubscribe?.();+ signal?.removeEventListener('abort', abort);+ clearTimeout(recheck);+ clearTimeout(deadline);+ resolve(settled);+ };+ const readState = async (): Promise<void> => {+ if (done) return;+ if (reading) {+ readRequested = true;+ return;+ }+ reading = true;+ clearTimeout(recheck);+ try {+ const job = await source.getGenerationSettlementState(conversationId);+ if (done) return;+ if (job == null || (epoch != null && epoch !== job.createdAt)) {+ finish(observedActive);+ return;+ }+ epoch = job.createdAt;+ const active =+ job.status === 'running' ||+ job.status === 'requires_action' ||+ job.terminalPersistencePending === true;+ if (!active) {+ finish(observedActive);+ return;+ }+ observedActive = true;+ } catch (error) {+ logger.warn(+ `[GenerationSettled] Failed to read generation state for ${conversationId}:`,+ error,+ );+ } finally {+ reading = false;+ if (!done) {+ if (readRequested) {+ readRequested = false;+ void readState();+ } else {+ recheck = setTimeout(() => void readState(), recheckMs);+ recheck.unref?.();+ }+ }+ }+ };+ if (maxWaitMs != null) {+ deadline = setTimeout(() => finish(false), maxWaitMs);+ deadline.unref?.();+ }+ signal?.addEventListener('abort', abort, { once: true });+ unsubscribe = source.onGenerationSettled((event) => {+ if (event.conversationId === conversationId) void readState();+ });+ if (done) unsubscribe();+ else void readState();+ });+}