Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🛟 fix: Promote Reconciled Compaction Snapshots to the Terminal Row (#16553)
Marco Beretta0bcb0aaec1d0
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.jsindex 6411c5c831..b582af6379 100644--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js@@ -17,6 +17,7 @@ const mockTenantStorageRun = jest.fn(async (context, callback) => {const mockSaveMessage = jest.fn();const mockGetConvo = jest.fn();const mockGetMessages = jest.fn();+const mockStampConvoLastResponse = jest.fn();const mockIsAgentTriggerPrincipalActive = jest.fn();const mockFilterPersistableAbortContent = jest.fn((content) => content);const mockCheckAndIncrementPendingRequest = jest.fn();@@ -58,10 +59,12 @@ jest.mock('@librechat/api', () => ({jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),applyForcedTemporaryRequest: jest.fn(),resolveResumableRetention: jest.requireActual('@librechat/api').resolveResumableRetention,- markAbortedCompactionContent: (...args) =>- jest.requireActual('@librechat/api').markAbortedCompactionContent(...args),- resolveDisconnectSnapshotMode: (...args) =>- jest.requireActual('@librechat/api').resolveDisconnectSnapshotMode(...args),+ planDisconnectSnapshot: (...args) =>+ jest.requireActual('@librechat/api').planDisconnectSnapshot(...args),+ createDisconnectSnapshotQueue: (...args) =>+ jest.requireActual('@librechat/api').createDisconnectSnapshotQueue(...args),+ announceDisconnectSnapshot: (...args) =>+ jest.requireActual('@librechat/api').announceDisconnectSnapshot(...args),settleExistingRowsBeforeErrorTurn: (...args) =>jest.requireActual('@librechat/api').settleExistingRowsBeforeErrorTurn(...args),sendEvent: jest.fn(),@@ -141,12 +144,19 @@ jest.mock('~/models', () => ({saveMessage: (...args) => mockSaveMessage(...args),getMessages: (...args) => mockGetMessages(...args),getConvo: (...args) => mockGetConvo(...args),+ stampConvoLastResponse: (...args) => mockStampConvoLastResponse(...args),isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),}));const AgentController = require('../request');describe('ResumableAgentController tenant context', () => {+ /** The anchor is persisted and no response or error row exists yet. */+ const anchorOnly = () =>+ mockGetMessages.mockImplementation(async (filter) =>+ filter.messageId === 'user-message' ? [{ _id: 'existing-anchor' }] : [],+ );+beforeEach(() => {jest.clearAllMocks();activeTenantContext = undefined;@@ -177,6 +187,7 @@ describe('ResumableAgentController tenant context', () => {fading: { v: 1, budgetTokens: 50_000, masked: true },};+ let lastAllSubscribersLeftHandler;const firePartialDisconnect = async (user,jobRecord = { createdAt: 1000, contextMeta: partialContextMeta },@@ -204,6 +215,7 @@ describe('ResumableAgentController tenant context', () => {on: jest.fn((event, handler) => {if (event === 'allSubscribersLeft') {allSubscribersLeftHandler = handler;+ lastAllSubscribersLeftHandler = handler;}}),},@@ -253,6 +265,7 @@ describe('ResumableAgentController tenant context', () => {it('carries the context meta the run published onto the partial response saved on disconnect', async () => {await firePartialDisconnect({ id: 'user-123', tenantId: 'tenant-a' });+ expect(mockStampConvoLastResponse).not.toHaveBeenCalled();expect(mockSaveMessage).toHaveBeenCalledWith(expect.any(Object),@@ -361,13 +374,17 @@ describe('ResumableAgentController tenant context', () => {expect(mockSaveMessage).not.toHaveBeenCalled();});- /** Ordinary turns keep the pre-change behavior exactly: their snapshot is- * the fallback row even when the job record has settled, because the- * terminal row write may still fail. */- it('still persists an ordinary partial save when the job record has settled', async () => {+ /** An ordinary turn's record cannot prove its response row landed (a+ * terminal transition without a write leaves the same record), so its+ * snapshot stays the fallback row, in its live shape. */+ it.each([+ ['a settled record', {}],+ ['a pending terminal write', { terminalPersistencePending: true }],+ ['a reconciliation frame', { finalEvent: JSON.stringify({ final: true, reconcile: true }) }],+ ])('keeps an ordinary partial save as the fallback behind %s', async (_label, record) => {await firePartialDisconnect({ id: 'user-123' },- { createdAt: 1000, status: 'error' },+ { createdAt: 1000, status: 'complete', ...record },{ aggregatedContent: [{ type: 'text', text: 'Partial response' }] },);@@ -375,4 +392,209 @@ describe('ResumableAgentController tenant context', () => {const [, savedMessage] = mockSaveMessage.mock.calls[0];expect(savedMessage).toMatchObject({ unfinished: true, error: false });});+ /** The terminal write failed and settled for a reconciliation frame: the+ * snapshot is the turn's only row, so it persists with the terminal+ * outcome (the typed failure for content with no summary) and envelope. */+ it('promotes a reconciled compaction snapshot to the terminal row', async () => {+ anchorOnly();+ await firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ {+ body: { compact: true },+ aggregatedContent: [{ type: 'think', think: 'Picking what to summarize' }],+ },+ );++ expect(mockSaveMessage).toHaveBeenCalledTimes(1);+ const [, savedMessage] = mockSaveMessage.mock.calls[0];+ expect(savedMessage).toMatchObject({ unfinished: false, error: true });+ expect(savedMessage.content).toEqual([+ { type: 'think', think: 'Picking what to summarize' },+ expect.objectContaining({ type: 'error', initiatedBy: 'user' }),+ ]);+ expect(mockStampConvoLastResponse).toHaveBeenCalledWith(+ 'user-123',+ 'conversation-123',+ 'response-message',+ );+ });+ /** A reconciliation frame does not prove the row is missing: an error turn+ * saved before it is a settled row the promotion must not overwrite. */+ it('withholds a reconciled snapshot when a settled response row exists', async () => {+ mockGetMessages.mockImplementation(async (filter) =>+ filter.messageId === 'response-message'+ ? [{ _id: 'error-row', unfinished: false }]+ : [{ _id: 'existing-anchor' }],+ );+ await firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ {+ body: { compact: true },+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],+ },+ );++ expect(mockSaveMessage).not.toHaveBeenCalled();+ });+ it('promotes a compaction snapshot on a later disconnect after a live save', async () => {+ mockGetMessages.mockImplementation(async (filter) =>+ filter.messageId === 'user-message' ? [{ _id: 'existing-anchor' }] : [],+ );+ await firePartialDisconnect(+ { id: 'user-123' },+ { createdAt: 1000, status: 'running' },+ { body: { compact: true } },+ );+ expect(mockSaveMessage).toHaveBeenCalledTimes(1);+ expect(mockSaveMessage.mock.calls[0][1]).toMatchObject({ unfinished: true });++ mockGenerationJobManager.getJobStore.mockReturnValue({+ getJob: jest.fn().mockResolvedValue({+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ }),+ });+ mockGetMessages.mockImplementation(async (filter) =>+ filter.messageId === 'user-message'+ ? [{ _id: 'existing-anchor' }]+ : [{ _id: 'live-row', unfinished: true }],+ );+ await lastAllSubscribersLeftHandler([{ type: 'text', text: 'Partial response' }]);++ expect(mockSaveMessage).toHaveBeenCalledTimes(2);+ expect(mockSaveMessage.mock.calls[1][1]).toMatchObject({ unfinished: false, error: true });+ });++ it('logs a failed row read while planning the snapshot instead of rejecting the listener', async () => {+ const readError = new Error('read failed');+ mockGetMessages.mockImplementation(async (filter) => {+ if (filter.messageId === 'user-message') {+ return [{ _id: 'existing-anchor' }];+ }+ throw readError;+ });+ await expect(+ firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ {+ body: { compact: true },+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],+ },+ ),+ ).resolves.not.toThrow();++ expect(mockSaveMessage).not.toHaveBeenCalled();+ expect(mockLogger.error).toHaveBeenCalledWith(+ '[ResumableAgentController] Error planning partial response:',+ readError,+ );+ });+ it('withholds a reconciled snapshot when the error turn saved its anchor-derived row', async () => {+ mockGetMessages.mockImplementation(+ async (filter) =>+ ({+ 'user-message': [{ _id: 'existing-anchor' }],+ 'user-message_': [{ _id: 'error-row', unfinished: false }],+ })[filter.messageId] ?? [],+ );+ await firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ {+ body: { compact: true },+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],+ },+ );++ expect(mockSaveMessage).not.toHaveBeenCalled();+ });++ /** Without a persisted anchor the promotion is withheld: the fallback must+ * not recreate the orphan the absent-anchor abort suppressed. */+ it('withholds a reconciled snapshot whose anchor was never persisted', async () => {+ await firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ {+ body: { compact: true },+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],+ },+ );++ expect(mockSaveMessage).not.toHaveBeenCalled();+ });++ /** An empty snapshot on a reconciled compaction still records the turn:+ * the terminal synthesis appends the typed failure from nothing. */+ it('synthesizes the terminal outcome for an empty reconciled compaction snapshot', async () => {+ anchorOnly();+ await firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ { body: { compact: true }, aggregatedContent: [] },+ );++ expect(mockSaveMessage).toHaveBeenCalledTimes(1);+ const [, savedMessage] = mockSaveMessage.mock.calls[0];+ expect(savedMessage).toMatchObject({ unfinished: false, error: true });+ expect(savedMessage.content).toEqual([+ expect.objectContaining({ type: 'error', initiatedBy: 'user' }),+ ]);+ });++ /** A completed claim's promoted snapshot settles as a finished row, not an+ * error. */+ it('settles a completed reconciled snapshot as a finished row', async () => {+ anchorOnly();+ await firePartialDisconnect(+ { id: 'user-123' },+ {+ createdAt: 1000,+ status: 'complete',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ {+ body: { compact: true },+ aggregatedContent: [+ {+ type: 'summary',+ content: [{ type: 'text', text: 'A finished checkpoint.' }],+ boundary: { messageId: 'm', contentIndex: 0 },+ },+ ],+ },+ );++ expect(mockSaveMessage).toHaveBeenCalledTimes(1);+ const [, savedMessage] = mockSaveMessage.mock.calls[0];+ expect(savedMessage).toMatchObject({ unfinished: false, error: false });+ });});diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex 0cd2f11aee..f3993086e3 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -273,10 +273,12 @@ jest.mock('@librechat/api', () => ({jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),applyForcedTemporaryRequest: jest.fn(),resolveResumableRetention: jest.requireActual('@librechat/api').resolveResumableRetention,- markAbortedCompactionContent: (...args) =>- jest.requireActual('@librechat/api').markAbortedCompactionContent(...args),- resolveDisconnectSnapshotMode: (...args) =>- jest.requireActual('@librechat/api').resolveDisconnectSnapshotMode(...args),+ planDisconnectSnapshot: (...args) =>+ jest.requireActual('@librechat/api').planDisconnectSnapshot(...args),+ createDisconnectSnapshotQueue: (...args) =>+ jest.requireActual('@librechat/api').createDisconnectSnapshotQueue(...args),+ announceDisconnectSnapshot: (...args) =>+ jest.requireActual('@librechat/api').announceDisconnectSnapshot(...args),settleExistingRowsBeforeErrorTurn: (...args) =>jest.requireActual('@librechat/api').settleExistingRowsBeforeErrorTurn(...args),sendEvent: jest.fn(),@@ -5191,6 +5193,138 @@ describe('ResumableAgentController resume metadata', () => {},);+ it('promotes a live compaction snapshot once its terminal write settles for reconciliation', async () => {+ const userMessage = {+ messageId: 'user-msg',+ parentMessageId: 'parent-msg',+ conversationId: 'conversation-123',+ text: '',+ };+ const textPart = { type: 'text', text: 'Partial summary' };+ let jobRecord = { createdAt: 1000, status: 'running' };+ let allSubscribersLeftHandler;+ mockGenerationJobManager.createJob.mockResolvedValue({+ createdAt: 1000,+ metadata: { providerExecutionId: 'provider-segment-1', providerDrained: true },+ readyPromise: Promise.resolve(),+ abortController: new AbortController(),+ emitter: {+ on: jest.fn((event, handler) => {+ if (event === 'allSubscribersLeft') {+ allSubscribersLeftHandler = handler;+ }+ }),+ },+ });+ mockGenerationJobManager.getJobStore.mockReturnValue({+ getJob: jest.fn(async () => jobRecord),+ getContentParts: jest.fn(async () => ({ content: [textPart] })),+ });+ mockGenerationJobManager.getResumeState.mockResolvedValue({+ conversationId: 'conversation-123',+ responseMessageId: 'response-msg',+ userMessage: { messageId: userMessage.messageId },+ });+ mockGenerationJobManager.publishTerminalClaim.mockImplementation(async (_claim, finalEvent) => {+ if (finalEvent == null) {+ jobRecord = {+ createdAt: 1000,+ status: 'complete',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ };+ }+ return { finalEvent, persistenceFailed: finalEvent == null };+ });+ mockGetMessages.mockImplementation(async ({ messageId }) => {+ if (messageId === userMessage.messageId) {+ return [{ _id: 'anchor' }];+ }+ return messageId === 'response-msg' ? [{ _id: 'live-row', unfinished: true }] : [];+ });+ let signalPromoted;+ const promoted = new Promise((resolve) => {+ signalPromoted = resolve;+ });+ let signalFinished;+ const finished = new Promise((resolve) => {+ signalFinished = resolve;+ });+ mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());+ mockSaveMessage.mockImplementation(async (_reqCtx, message) => {+ if (message?.messageId === userMessage.messageId) {+ return {};+ }+ if (message?.unfinished === false && message?.error === false) {+ signalPromoted(message);+ return {};+ }+ /** The live snapshot lands; the completed response write does not. */+ return message?.unfinished === true ? {} : undefined;+ });+ const client = {+ options: {},+ savedMessageIds: new Set(['response-msg']),+ skipSaveUserMessage: false,+ sendMessage: jest.fn(async (_text, options) => {+ options.onStart(userMessage, 'response-msg');+ await allSubscribersLeftHandler([textPart]);+ return {+ messageId: 'response-msg',+ parentMessageId: userMessage.messageId,+ conversationId: 'conversation-123',+ content: [textPart],+ databasePromise: Promise.resolve({+ conversation: { conversationId: 'conversation-123', title: 'Existing' },+ }),+ };+ }),+ };+ const req = {+ user: { id: 'user-123' },+ body: {+ text: '',+ compact: true,+ messageId: userMessage.messageId,+ parentMessageId: userMessage.messageId,+ conversationId: 'conversation-123',+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },+ },+ config: {},+ };++ await AgentController(+ req,+ createResumableResponse(),+ jest.fn(),+ jest.fn().mockResolvedValue({ client }),+ null,+ );+ const [promotedRow] = await Promise.all([promoted, finished]);++ const snapshotWrites = mockSaveMessage.mock.calls+ .map(([, message]) => message)+ .filter((message) => message.messageId === 'response-msg' && message.unfinished != null);+ expect(snapshotWrites[0]).toMatchObject({ unfinished: true, error: false });+ expect(promotedRow).toMatchObject({+ messageId: 'response-msg',+ parentMessageId: userMessage.messageId,+ unfinished: false,+ error: false,+ });+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(+ expect.anything(),+ null,+ );+ /** Terminal cleanup releases the content and resume state the plan reads. */+ const promotedOrder =+ mockSaveMessage.mock.invocationCallOrder[+ mockSaveMessage.mock.calls.findIndex(([, message]) => message === promotedRow)+ ];+ expect(promotedOrder).toBeLessThan(+ mockGenerationJobManager.finishTerminalJob.mock.invocationCallOrder[0],+ );+ });+it('suppresses the completed response write when Stop already won terminal ownership', async () => {const userMessage = {messageId: 'user-msg',diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex d43a1f9849..8750242ffd 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -63,10 +63,11 @@ const {getUserFacingGenerationError,stampPreliminaryPrivateTextMessage,announceReply,+ announceDisconnectSnapshot,announceErrorTurn,settleExistingRowsBeforeErrorTurn,- resolveDisconnectSnapshotMode,- markAbortedCompactionContent,+ planDisconnectSnapshot,+ createDisconnectSnapshotQueue,isTemporaryTurn,getPreliminaryResponseMessageId,} = require('@librechat/api');@@ -1903,21 +1904,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit* actual response messageId isn't available yet. The final response save will* overwrite this with the complete response using the same messageId pattern.*/- job.emitter.on('allSubscribersLeft', async (aggregatedContent) => {- if (partialResponseSaved || !aggregatedContent || aggregatedContent.length === 0) {- return;- }-- /** The run is still live here: mark what streamed, but leave the outcome- * to whichever path settles the turn (the terminal abort synthesizes- * the typed failure a stopped compaction with no summary needs). */- const persistableContent = markAbortedCompactionContent(- filterPersistableAbortContent(aggregatedContent),- isCompaction,- { synthesizeFailure: false },- );- if (persistableContent.length === 0) {- logger.debug('[ResumableAgentController] No persistable content to save partial response');+ const persistDisconnectSnapshot = async (aggregatedContent) => {+ /** Empty content is rejected only after the snapshot mode is known: a+ * reconciled compaction synthesizes its terminal outcome from nothing,+ * while a live run has nothing persistable. Whether an earlier live+ * save still allows a write is part of that plan. */+ if (!aggregatedContent) {return;}@@ -1930,8 +1922,34 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitreturn;}- partialResponseSaved = true;+ /** Whether and how this snapshot persists is planned in @librechat/api,+ * inside the request's tenant context like the write below. */+ const responseMessageId =+ resumeState.responseMessageId || `${resumeState.userMessage.messageId}_`;const responseConversationId = resumeState.conversationId || conversationId;+ const planSnapshot = () =>+ planDisconnectSnapshot(isCompaction, {+ content: filterPersistableAbortContent(aggregatedContent),+ jobRecord,+ jobCreatedAt,+ userId,+ conversationId: responseConversationId,+ anchorMessageId: resumeState.userMessage.messageId,+ responseMessageId,+ liveSaved: partialResponseSaved,+ getMessages,+ onReadError: (error) =>+ logger.error('[ResumableAgentController] Error planning partial response:', error),+ });+ const snapshot = tenantId+ ? await tenantStorage.run({ tenantId, userId }, planSnapshot)+ : await planSnapshot();+ if (!snapshot.write) {+ logger.debug('[ResumableAgentController] No partial response to save on disconnect');+ return;+ }++ partialResponseSaved = true;/** The run publishes its calibration and fading tiers onto the job; a* partial response saved on disconnect must carry them like the Stop and* pause paths do, or a turn continued from it re-derives its provider@@ -1939,22 +1957,16 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit* record is the source, since the client-facing resume snapshot never* carries server-private state. */const contextMeta = jobRecord?.createdAt === jobCreatedAt ? jobRecord.contextMeta : undefined;- if (resolveDisconnectSnapshotMode(isCompaction, jobRecord, jobCreatedAt) === 'skip') {- logger.debug(- '[ResumableAgentController] Skipping compaction partial save for a settled job',- );- return;- }try {const partialMessage = {- messageId: resumeState.responseMessageId || `${resumeState.userMessage.messageId}_`,+ messageId: responseMessageId,conversationId: responseConversationId,parentMessageId: resumeState.userMessage.messageId,sender: client?.sender ?? 'AI',- content: persistableContent,- unfinished: true,- error: false,+ content: snapshot.content,+ unfinished: snapshot.unfinished,+ error: snapshot.error,isCreatedByUser: false,user: userId,endpoint: endpointOption.endpoint,@@ -1967,14 +1979,15 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitpartialMessage.agent_id = req.body.agent_id;}+ const isTemporary =+ req?._agentEventBindingRetention?.isTemporary ??+ req?.resolvedConversation?.isTemporary ??+ req?.body?.isTemporary;const savePartialMessage = () =>saveMessage({userId,- isTemporary:- req?._agentEventBindingRetention?.isTemporary ??- req?.resolvedConversation?.isTemporary ??- req?.body?.isTemporary,+ isTemporary,expiredAt:req?._agentEventBindingRetention?.expiredAt ?? req?.resolvedConversation?.expiredAt,interfaceConfig: req?.config?.interfaceConfig,@@ -1992,15 +2005,40 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitthrow new Error('Partial response could not be persisted after disconnect');}+ const announcePromotion = () =>+ announceDisconnectSnapshot({ stampConvoLastResponse }, snapshot, {+ userId,+ conversationId: responseConversationId,+ messageId: responseMessageId,+ isTemporary,+ context: 'ResumableAgentController - promoted disconnect snapshot',+ });+ await (tenantId+ ? tenantStorage.run({ tenantId, userId }, announcePromotion)+ : announcePromotion());+logger.debug(- `[ResumableAgentController] Saved partial response for ${streamId}, content parts: ${persistableContent.length}`,+ `[ResumableAgentController] Saved partial response for ${streamId}, content parts: ${snapshot.content.length}`,);} catch (error) {logger.error('[ResumableAgentController] Error saving partial response:', error);// Reset flag so we can try again if subscribers reconnect and leave againpartialResponseSaved = false;}+ };+ const disconnectSnapshots = createDisconnectSnapshotQueue({+ isCompaction,+ persist: persistDisconnectSnapshot,+ readContent: async () =>+ (await GenerationJobManager.getJobStore().getContentParts(streamId, jobCreatedAt))+ ?.content ?? [],+ onSettleError: (error) =>+ logger.error(+ '[ResumableAgentController] Error settling the saved partial response:',+ error,+ ),});+ job.emitter.on('allSubscribersLeft', disconnectSnapshots.onDisconnect);/** @type {{ client: TAgentClient; userMCPAuthMap?: Record<string, Record<string, string>> }} */const result = await initializeClient({@@ -2204,6 +2242,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitreturn;}try {+ await disconnectSnapshots.settle();await GenerationJobManager.finishTerminalJob(terminalClaim);} finally {terminalClaimFinished = true;@@ -3408,6 +3447,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitinitialAgentId: verifiedInitialAgentId,}),),+ beforeTerminalCleanup: disconnectSnapshots.settle,})) === true;/** A true completion means this owner won the terminal CAS and* the beforeErrorPublication barrier above finished. Only thatdiff --git a/e2e/specs/mock/scenarios/compaction-snapshot-modes.spec.ts b/e2e/specs/mock/scenarios/compaction-snapshot-modes.spec.tsnew file mode 100644index 0000000000..2ae0745ef2--- /dev/null+++ b/e2e/specs/mock/scenarios/compaction-snapshot-modes.spec.ts@@ -0,0 +1,106 @@+import { expect, test } from '@playwright/test';+import { randomUUID } from 'node:crypto';+import { getE2EUser } from '../../../setup/user';+import { deleteConversations, deleteMessagesByConversation, seedMessages, withMongo } from '../db';+import { messagesView, sendMessageAndWaitForCompletion } from '../helpers';++const userEmail = getE2EUser().email;+const LABEL_SERVER = `http://127.0.0.1:${process.env.E2E_LABEL_PORT || '8889'}`;++type Row = Record<string, unknown>;++async function cleanup(conversationId: string) {+ await deleteMessagesByConversation([conversationId]);+ await deleteConversations([conversationId]);+}++function findRow(filter: Row): Promise<Row | null> {+ return withMongo((db) => db.collection('messages').findOne(filter));+}++test.describe('compaction snapshot modes', () => {+ test.afterEach(async ({ request }) => {+ const response = await request.post(`${LABEL_SERVER}/__e2e/reset`);+ expect(response.ok()).toBeTruthy();+ });++ /* The last viewer leaving a live compaction saves its snapshot in the live+ shape; the run then finishes on its own and its terminal row replaces the+ snapshot, so the reopened conversation shows a settled compaction. */+ test('a compaction whose viewer leaves mid-run still settles after it finishes @scenario:abandoned-compaction-settles-after-the-run', async ({+ page,+ request,+ }) => {+ test.setTimeout(150_000);+ await page.goto('/c/new');+ await sendMessageAndWaitForCompletion(page, 'tell me about abandoned compactions');+ const conversationId = new URL(page.url()).pathname.replace('/c/', '');+ expect(conversationId).not.toBe('new');++ try {+ const answer = await withMongo((db) =>+ db+ .collection('messages')+ .findOne({ conversationId, isCreatedByUser: false }, { sort: { createdAt: -1 } }),+ );+ expect(answer?.messageId).toBeTruthy();++ const leafId = randomUUID();+ const leafText = 'Compact this and walk away';+ await seedMessages(userEmail, conversationId, [+ {+ messageId: leafId,+ parentMessageId: answer?.messageId as string,+ text: leafText,+ isCreatedByUser: true,+ sender: 'User',+ },+ ]);++ const behavior = await request.post(`${LABEL_SERVER}/__e2e/behavior`, {+ data: { mode: 'ok', delayMs: 8_000 },+ });+ expect(behavior.ok()).toBeTruthy();++ await page.goto(`/c/${conversationId}`);+ await expect(messagesView(page).getByText(leafText)).toBeVisible();+ await page.getByTestId('token-usage').click();+ await page.getByRole('button', { name: 'Compact context' }).click();+ await expect(page.getByTestId('stop-generation-button')).toBeVisible({ timeout: 20_000 });++ /* Closing the only viewer drops every subscriber while the summary runs. */+ const context = page.context();+ await page.close();++ let compaction: Row | null = null;+ await expect+ .poll(+ async () => {+ compaction = await findRow({+ conversationId,+ parentMessageId: leafId,+ isCreatedByUser: false,+ });+ return compaction != null && compaction.unfinished !== true;+ },+ { timeout: 60_000, intervals: [2_000] },+ )+ .toBeTruthy();+ expect((compaction as Row | null)?.error).not.toBe(true);++ const leaf = await findRow({ conversationId, messageId: leafId });+ expect(leaf?.isCreatedByUser).toBe(true);+ expect(leaf?.text).toBe(leafText);++ const viewer = await context.newPage();+ await viewer.goto(`/c/${conversationId}`);+ await expect(messagesView(viewer).getByText(leafText)).toBeVisible({ timeout: 20_000 });+ const row = viewer.locator(`[id="${(compaction as Row | null)?.messageId as string}"]`);+ await expect(row).toBeVisible({ timeout: 20_000 });+ await expect(row.getByText('Summarizing...')).toHaveCount(0);+ await expect(row.getByText('Could not compact the context', { exact: false })).toHaveCount(0);+ } finally {+ await cleanup(conversationId);+ }+ });+});diff --git a/packages/api/src/agents/compaction.spec.ts b/packages/api/src/agents/compaction.spec.tsindex 2e58d808f9..9a75153f4f 100644--- a/packages/api/src/agents/compaction.spec.ts+++ b/packages/api/src/agents/compaction.spec.ts@@ -20,7 +20,11 @@ import {persistFinalizedCompactionTurn,isSettledJobRecord,resolveDisconnectSnapshotMode,+ resolveReconciledSnapshotEnvelope,planAbortedTurnPersistence,+ planDisconnectSnapshot,+ createDisconnectSnapshotQueue,+ announceDisconnectSnapshot,resolveAbortedTurnPersistence,resolveAbortedTurnAnchorDecision,settleExistingRowsBeforeErrorTurn,@@ -31,6 +35,7 @@ import {stripUnusableSummaryParts,getLatestEventActorSummary,} from './compaction';+import { MAX_RESPONSE_MESSAGE_ID_LENGTH } from '~/utils/message';const index = [{@@ -1434,6 +1439,25 @@ describe('isSettledJobRecord', () => {).toBe(false);});+ /** A failed terminal write clears the marker while publishing a+ * reconciliation frame: no row was persisted, so the streamed snapshot+ * stays the turn's only fallback. */+ it('treats a record whose durable final event is a reconciliation frame as unsettled', () => {+ const reconciled = {+ createdAt: 1000,+ status: 'aborted',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ };+ const settled = {+ createdAt: 1000,+ status: 'aborted',+ finalEvent: JSON.stringify({ final: true }),+ };++ expect(isSettledJobRecord(reconciled)).toBe(false);+ expect(isSettledJobRecord(settled)).toBe(true);+ });+it('leaves live and missing records unsettled', () => {expect(isSettledJobRecord({ createdAt: 1000, status: 'running' })).toBe(false);expect(isSettledJobRecord({ createdAt: 1000, status: 'requires_action' })).toBe(false);@@ -1451,26 +1475,442 @@ describe('isSettledJobRecord', () => {describe('resolveDisconnectSnapshotMode', () => {it.each(['complete', 'error', 'aborted'])('withholds the snapshot of a %s compaction job',- (status) => {- expect(resolveDisconnectSnapshotMode(true, { createdAt: 1000, status }, 1000)).toBe('skip');+ async (status) => {+ await expect(+ resolveDisconnectSnapshotMode(true, { createdAt: 1000, status }, 1000),+ ).resolves.toBe('skip');},);- it('writes the snapshot for a live, missing, or other-epoch record', () => {- expect(resolveDisconnectSnapshotMode(true, { createdAt: 1000, status: 'running' }, 1000)).toBe(- 'live',+ it('writes the snapshot for a live, missing, or other-epoch record', async () => {+ await expect(+ resolveDisconnectSnapshotMode(true, { createdAt: 1000, status: 'running' }, 1000),+ ).resolves.toBe('live');+ await expect(resolveDisconnectSnapshotMode(true, null, 1000)).resolves.toBe('live');+ await expect(+ resolveDisconnectSnapshotMode(true, { createdAt: 2000, status: 'error' }, 1000),+ ).resolves.toBe('live');+ });++ /** An ordinary turn's record cannot prove its response row landed, so its+ * snapshot stays the fallback whatever the record says. */+ it('writes the snapshot of a settled ordinary turn', async () => {+ await expect(+ resolveDisconnectSnapshotMode(false, { createdAt: 1000, status: 'complete' }, 1000),+ ).resolves.toBe('live');+ });++ /** Only a compaction is promoted: an ordinary turn whose terminal write+ * reconciled keeps its live snapshot as the fallback row. */+ it('keeps a reconciled ordinary turn on its live snapshot', async () => {+ const anchorExists = jest.fn(async () => true);+ await expect(+ resolveDisconnectSnapshotMode(+ false,+ {+ createdAt: 1000,+ status: 'complete',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ },+ 1000,+ { anchorExists },+ ),+ ).resolves.toBe('live');+ expect(anchorExists).not.toHaveBeenCalled();+ });++ const reconciled = () => ({+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ });++ /** The terminal write failed and settled for a reconciliation frame: the+ * snapshot is promoted to the turn's terminal row, because no other row+ * will ever be persisted for it. */+ it('promotes a reconciled compaction snapshot to the terminal row', async () => {+ await expect(resolveDisconnectSnapshotMode(true, reconciled(), 1000)).resolves.toBe('terminal');+ });++ /** A reconciliation frame can follow a saved error turn: a settled+ * response row withholds the promotion instead of being overwritten. */+ it('withholds a reconciled snapshot when the response row is settled', async () => {+ await expect(+ resolveDisconnectSnapshotMode(true, reconciled(), 1000, {+ responseRowSettled: async () => true,+ }),+ ).resolves.toBe('skip');+ });++ it('keeps the live snapshot for another epoch reconciliation frame', async () => {+ const anchorExists = jest.fn(async () => true);+ await expect(+ resolveDisconnectSnapshotMode(true, { ...reconciled(), createdAt: 2000 }, 1000, {+ anchorExists,+ }),+ ).resolves.toBe('live');+ expect(anchorExists).not.toHaveBeenCalled();+ });++ /** The promotion must not recreate the orphan an absent-anchor abort+ * deliberately withheld: without a persisted anchor there is nothing to+ * hang the terminal row on. */+ it('withholds a reconciled snapshot whose anchor was never persisted', async () => {+ await expect(+ resolveDisconnectSnapshotMode(true, reconciled(), 1000, {+ anchorExists: async () => false,+ }),+ ).resolves.toBe('skip');+ });+});++describe('resolveReconciledSnapshotEnvelope', () => {+ /** The abort route writes a stopped compaction settled, so its+ * reconciled fallback must not reopen it as a live row. */+ it('settles an aborted claim like the abort route writes a stopped compaction', () => {+ expect(resolveReconciledSnapshotEnvelope('aborted')).toEqual({+ unfinished: false,+ error: false,+ });+ });++ it('settles a completed claim as a finished row', () => {+ expect(resolveReconciledSnapshotEnvelope('complete')).toEqual({+ unfinished: false,+ error: false,+ });+ });++ it('settles everything else with the error envelope', () => {+ expect(resolveReconciledSnapshotEnvelope('error')).toEqual({+ unfinished: false,+ error: true,+ });+ expect(resolveReconciledSnapshotEnvelope(undefined)).toEqual({+ unfinished: false,+ error: true,+ });+ });+});++describe('planDisconnectSnapshot', () => {+ const reconciled = {+ createdAt: 1000,+ status: 'error',+ finalEvent: JSON.stringify({ final: true, reconcile: true }),+ };+ const plan = (+ isCompaction: boolean,+ jobRecord: Parameters<typeof planDisconnectSnapshot>[1]['jobRecord'],+ content: TMessageContentParts[],+ rows: Record<string, Array<{ messageId: string; unfinished?: boolean }>>,+ ) => {+ const getMessages = jest.fn(+ async (filter: { messageId: string }) => rows[filter.messageId] ?? [],);- expect(resolveDisconnectSnapshotMode(true, null, 1000)).toBe('live');- expect(resolveDisconnectSnapshotMode(true, { createdAt: 2000, status: 'error' }, 1000)).toBe(- 'live',+ return {+ getMessages,+ result: planDisconnectSnapshot(isCompaction, {+ content,+ jobRecord,+ jobCreatedAt: 1000,+ userId: 'user-1',+ conversationId: 'convo-1',+ anchorMessageId: 'anchor',+ responseMessageId: 'response',+ getMessages,+ }),+ };+ };+ const text = { type: ContentTypes.TEXT, text: 'Partial' } as TMessageContentParts;++ it('writes a live snapshot in the live shape without reading any row', async () => {+ const { result, getMessages } = plan(+ false,+ { createdAt: 1000, status: 'complete' },+ [text],+ {},);++ await expect(result).resolves.toEqual({+ write: true,+ content: [text],+ unfinished: true,+ error: false,+ });+ expect(getMessages).not.toHaveBeenCalled();});- /** An ordinary turn's snapshot is the fallback row its terminal write may- * still need, so it is written whatever the record says. */- it('writes the snapshot of a settled ordinary turn', () => {- expect(- resolveDisconnectSnapshotMode(false, { createdAt: 1000, status: 'complete' }, 1000),- ).toBe('live');+ it('writes nothing for an empty live snapshot', async () => {+ const { result } = plan(true, { createdAt: 1000, status: 'running' }, [], {});++ await expect(result).resolves.toEqual({ write: false });+ });++ it('promotes a reconciled compaction under its anchor with the terminal envelope', async () => {+ const { result } = plan(true, reconciled, [], { anchor: [{ messageId: 'anchor' }] });+ const planned = await result;++ expect(planned).toMatchObject({ write: true, unfinished: false, error: true });+ expect(planned.write && planned.content).toEqual([+ expect.objectContaining({ type: ContentTypes.ERROR, initiatedBy: 'user' }),+ ]);+ });++ it('promotes over an earlier live snapshot of the same turn', async () => {+ const { result } = plan(true, reconciled, [text], {+ anchor: [{ messageId: 'anchor' }],+ response: [{ messageId: 'response', unfinished: true }],+ });++ await expect(result).resolves.toMatchObject({ write: true, unfinished: false });+ });++ it('withholds the promotion when a settled response row exists', async () => {+ const { result } = plan(true, reconciled, [text], {+ anchor: [{ messageId: 'anchor' }],+ response: [{ messageId: 'response', unfinished: false }],+ });++ await expect(result).resolves.toEqual({ write: false });+ });++ it('reports a failed row read and writes nothing', async () => {+ const readError = new Error('read failed');+ const onReadError = jest.fn();++ await expect(+ planDisconnectSnapshot(true, {+ content: [text],+ jobRecord: reconciled,+ jobCreatedAt: 1000,+ userId: 'user-1',+ conversationId: 'convo-1',+ anchorMessageId: 'anchor',+ responseMessageId: 'response',+ getMessages: jest.fn(async () => {+ throw readError;+ }),+ onReadError,+ }),+ ).resolves.toEqual({ write: false });+ expect(onReadError).toHaveBeenCalledWith(readError);+ });++ it('writes a live snapshot once but still promotes after it', async () => {+ const live = planDisconnectSnapshot(true, {+ content: [text],+ jobRecord: { createdAt: 1000, status: 'running' },+ jobCreatedAt: 1000,+ userId: 'user-1',+ conversationId: 'convo-1',+ anchorMessageId: 'anchor',+ responseMessageId: 'response',+ liveSaved: true,+ getMessages: jest.fn(async () => []),+ });+ await expect(live).resolves.toEqual({ write: false });++ const getMessages = jest.fn(async (filter: { messageId: string }) =>+ filter.messageId === 'anchor'+ ? [{ messageId: 'anchor' }]+ : [{ messageId: filter.messageId, unfinished: true }],+ );+ const promoted = planDisconnectSnapshot(true, {+ content: [text],+ jobRecord: reconciled,+ jobCreatedAt: 1000,+ userId: 'user-1',+ conversationId: 'convo-1',+ anchorMessageId: 'anchor',+ responseMessageId: 'response',+ liveSaved: true,+ getMessages,+ });+ await expect(promoted).resolves.toMatchObject({ write: true, unfinished: false });+ });++ it('withholds the promotion when the failed-turn path already saved its error row', async () => {+ const { result, getMessages } = plan(true, reconciled, [text], {+ anchor: [{ messageId: 'anchor' }],+ anchor_: [{ messageId: 'anchor_', unfinished: false }],+ });++ await expect(result).resolves.toEqual({ write: false });+ expect(getMessages).toHaveBeenCalledWith(+ expect.objectContaining({ messageId: 'anchor_' }),+ '_id unfinished',+ );+ });++ it('still promotes when the derived error row is only an unfinished snapshot', async () => {+ const { result } = plan(true, reconciled, [text], {+ anchor: [{ messageId: 'anchor' }],+ anchor_: [{ messageId: 'anchor_', unfinished: true }],+ });++ await expect(result).resolves.toMatchObject({ write: true, unfinished: false });+ });++ it('never treats an anchor ending in one underscore as its own terminal row', async () => {+ const getMessages = jest.fn(async (filter: { messageId: string }) =>+ filter.messageId === 'anchor_' ? [{ messageId: 'anchor_', unfinished: false }] : [],+ );++ await expect(+ planDisconnectSnapshot(true, {+ content: [text],+ jobRecord: reconciled,+ jobCreatedAt: 1000,+ userId: 'user-1',+ conversationId: 'convo-1',+ anchorMessageId: 'anchor_',+ responseMessageId: 'response',+ getMessages,+ }),+ ).resolves.toMatchObject({ write: true, unfinished: false });+ expect(getMessages).toHaveBeenCalledWith(+ expect.objectContaining({ messageId: 'response' }),+ '_id unfinished',+ );+ expect(getMessages).not.toHaveBeenCalledWith(+ expect.objectContaining({ messageId: 'anchor_' }),+ '_id unfinished',+ );+ });++ it('reads only the response row when a full-length anchor leaves no room for the suffix', async () => {+ const anchorMessageId = 'a'.repeat(MAX_RESPONSE_MESSAGE_ID_LENGTH);+ const getMessages = jest.fn(async (filter: { messageId: string }) =>+ filter.messageId === anchorMessageId ? [{ messageId: anchorMessageId }] : [],+ );++ await expect(+ planDisconnectSnapshot(true, {+ content: [text],+ jobRecord: reconciled,+ jobCreatedAt: 1000,+ userId: 'user-1',+ conversationId: 'convo-1',+ anchorMessageId,+ responseMessageId: 'response',+ getMessages,+ }),+ ).resolves.toMatchObject({ write: true, unfinished: false });+ expect(getMessages.mock.calls.map(([filter]) => filter.messageId).sort()).toEqual(+ [anchorMessageId, 'response'].sort(),+ );+ });+});++describe('createDisconnectSnapshotQueue', () => {+ const text = { type: ContentTypes.TEXT, text: 'Partial' } as TMessageContentParts;+ const stored = { type: ContentTypes.TEXT, text: 'Stored' } as TMessageContentParts;++ it('plans once more from the stored content when a disconnected compaction settles', async () => {+ const persist = jest.fn(async () => undefined);+ const queue = createDisconnectSnapshotQueue({+ isCompaction: true,+ persist,+ readContent: async () => [stored],+ });++ await queue.onDisconnect([]);+ await queue.settle();++ expect(persist.mock.calls).toEqual([[[]], [[stored]]]);+ });++ it('does not plan again when nobody disconnected or the turn is not a compaction', async () => {+ const persist = jest.fn(async () => undefined);+ const readContent = jest.fn(async () => [stored]);++ await createDisconnectSnapshotQueue({ isCompaction: true, persist, readContent }).settle();+ const ordinary = createDisconnectSnapshotQueue({ isCompaction: false, persist, readContent });+ await ordinary.onDisconnect([text]);+ await ordinary.settle();++ expect(persist.mock.calls).toEqual([[[text]]]);+ expect(readContent).not.toHaveBeenCalled();+ });++ it('waits for an outstanding snapshot write before the settle plan starts', async () => {+ const order: string[] = [];+ let finishLive: () => void = () => undefined;+ const persist = jest.fn(async (content: TMessageContentParts[]) => {+ const label = content[0] === stored ? 'settle' : 'live';+ order.push(`${label}:start`);+ if (label === 'live') {+ await new Promise<void>((resolve) => {+ finishLive = resolve;+ });+ }+ order.push(`${label}:end`);+ });+ const queue = createDisconnectSnapshotQueue({+ isCompaction: true,+ persist,+ readContent: async () => [stored],+ });++ const live = queue.onDisconnect([text]);+ const settled = queue.settle();+ await Promise.resolve();+ finishLive();+ await Promise.all([live, settled]);++ expect(order).toEqual(['live:start', 'live:end', 'settle:start', 'settle:end']);+ });++ it('reports a failed settle plan and still runs after a failed live write', async () => {+ const liveError = new Error('live failed');+ const settleError = new Error('read failed');+ const onSettleError = jest.fn();+ const queue = createDisconnectSnapshotQueue({+ isCompaction: true,+ persist: jest.fn(async () => {+ throw liveError;+ }),+ readContent: async () => {+ throw settleError;+ },+ onSettleError,+ });++ await expect(queue.onDisconnect([text])).rejects.toBe(liveError);+ await expect(queue.settle()).resolves.toBeUndefined();+ expect(onSettleError).toHaveBeenCalledWith(settleError);+ });+});++describe('announceDisconnectSnapshot', () => {+ const announcement = {+ userId: 'user-1',+ conversationId: 'convo-1',+ messageId: 'response',+ context: 'test',+ };++ it('stamps a promoted terminal snapshot on its conversation', async () => {+ const stampConvoLastResponse = jest.fn(async () => null);+ await announceDisconnectSnapshot(+ { stampConvoLastResponse },+ { write: true, content: [], unfinished: false, error: true },+ announcement,+ );++ expect(stampConvoLastResponse).toHaveBeenCalledWith('user-1', 'convo-1', 'response');+ });++ it('leaves a live or unwritten snapshot unannounced', async () => {+ const stampConvoLastResponse = jest.fn(async () => null);+ await announceDisconnectSnapshot(+ { stampConvoLastResponse },+ { write: true, content: [], unfinished: true, error: false },+ announcement,+ );+ await announceDisconnectSnapshot({ stampConvoLastResponse }, { write: false }, announcement);++ expect(stampConvoLastResponse).not.toHaveBeenCalled();});});diff --git a/packages/api/src/agents/compaction.ts b/packages/api/src/agents/compaction.tsindex cbf3c5e66f..c4afa57a8a 100644--- a/packages/api/src/agents/compaction.ts+++ b/packages/api/src/agents/compaction.ts@@ -18,7 +18,10 @@ import type {} from '@librechat/data-schemas';import type { SummaryContentPart, TMessageContentParts } from 'librechat-data-provider';import type { IAgentEventActorSummary } from '@librechat/data-schemas';+import type { ErrorTurnAnnouncement, ErrorTurnStampStore } from '~/conversations/announce';+import { getPreliminaryResponseMessageId } from '~/utils/message';import { createAgentEventActorSummary } from './compatibility';+import { announceErrorTurn } from '~/conversations/announce';/** Text of a summary content part, in any persisted shape — `content` blocks* today, a string `content` or a bare `text` on rows written before them.@@ -277,16 +280,33 @@ export function markAbortedCompactionContent(return marked;}+/** Whether a job record's durable final event is a reconciliation frame: the+ * conservative substitute published when the terminal row write failed, so+ * no message row backs the terminal claim. */+function hasDurableReconcileFrame(finalEvent: unknown): boolean {+ if (typeof finalEvent !== 'string' || finalEvent.length === 0) {+ return false;+ }+ try {+ const parsed = JSON.parse(finalEvent) as { reconcile?: unknown } | null;+ return parsed?.reconcile === true;+ } catch {+ return false;+ }+}+/** Whether a job record has reached a status whose path owns the turn's final* row (completion, error, or abort): the disconnect snapshot must not be* written over it, or the settled row reopens as an unfinished response.- * Only a same-epoch record is trusted. */+ * Only a same-epoch record is trusted, and only one whose terminal write+ * actually landed. */export function isSettledJobRecord(jobRecord:| {createdAt?: number;status?: string;terminalPersistencePending?: boolean;+ finalEvent?: string;}| null| undefined,@@ -301,6 +321,12 @@ export function isSettledJobRecord(* that write fails. */return false;}+ if (hasDurableReconcileFrame(jobRecord.finalEvent)) {+ /** The terminal write failed and a reconciliation frame was published in+ * its place: nothing was persisted for the turn, so the streamed+ * snapshot remains its only row. */+ return false;+ }return (jobRecord.status === 'complete' ||jobRecord.status === 'error' ||@@ -308,31 +334,257 @@ export function isSettledJobRecord();}-/** How the last-subscriber disconnect may persist this turn's snapshot. */+/** How a disconnect may persist this turn's snapshot. */export type DisconnectSnapshotMode =- /** The run is still live: the snapshot is written as the fallback row. */+ /** The run is still live: the snapshot keeps the live shape. */| 'live'- /** A settling path owns the final row: the snapshot is withheld so it- * cannot reopen the settled turn as an unfinished response. */+ /** The terminal write failed and settled for a reconciliation frame: the+ * snapshot is the turn's only row, so it persists with the terminal+ * outcome and envelope. */+ | 'terminal'+ /** A durable terminal row exists: the snapshot is withheld so it cannot+ * reopen the settled turn. */| 'skip';+/** The row flags a promoted compaction snapshot settles with, keyed by the+ * reconciled claim's status: a stopped compaction is settled like the abort+ * route writes it (nothing continues it), a completed run is a finished row,+ * and everything else takes the error envelope. */+export function resolveReconciledSnapshotEnvelope(status: unknown): {+ unfinished: boolean;+ error: boolean;+} {+ if (status === 'aborted' || status === 'complete') {+ return { unfinished: false, error: false };+ }+ return { unfinished: false, error: true };+}+/**- * How the last-subscriber disconnect may persist this turn's snapshot, read- * from the same-epoch job record the caller already loaded. A compaction- * whose settling path owns the final row withholds the snapshot; the guard- * reads the record that path writes, so the remaining window is the path's- * own commit span. Ordinary turns keep writing their fallback row, settled- * or not.+ * How the last-subscriber disconnect may persist this turn's snapshot. An+ * ordinary turn always keeps its live snapshot as the fallback row: its job+ * record cannot prove a response row landed. A compaction whose settling path+ * (completion, error, abort) durably owns the final row withholds the+ * snapshot, so it cannot reopen the settled turn. A same-epoch compaction+ * whose terminal claim settled for a reconciliation frame is promoted to the+ * terminal row, but a reconciliation frame does not prove the row is missing+ * (an error turn can land before it), so the response row is read first and a+ * settled one withholds the promotion; the anchor must be persisted too, or+ * the promotion would recreate the orphan an absent-anchor abort withheld.*/-export function resolveDisconnectSnapshotMode(+export async function resolveDisconnectSnapshotMode(isCompaction: boolean,- jobRecord: { createdAt?: number; status?: string } | null | undefined,- jobCreatedAt?: number,-): DisconnectSnapshotMode {+ jobRecord:+ | {+ createdAt?: number;+ status?: string;+ terminalPersistencePending?: boolean;+ finalEvent?: string;+ }+ | null+ | undefined,+ jobCreatedAt: number | undefined,+ {+ anchorExists = async () => true,+ responseRowSettled = async () => false,+ }: {+ anchorExists?: () => Promise<boolean>;+ /** Whether a finished (not `unfinished`) response row already exists. */+ responseRowSettled?: () => Promise<boolean>;+ } = {},+): Promise<DisconnectSnapshotMode> {if (!isCompaction) {return 'live';}- return isSettledJobRecord(jobRecord, jobCreatedAt) ? 'skip' : 'live';+ if (isSettledJobRecord(jobRecord, jobCreatedAt)) {+ return 'skip';+ }+ /** Another epoch's reconciliation frame belongs to a different+ * generation and never promotes this one's snapshot. */+ const sameEpoch = jobCreatedAt == null || jobRecord?.createdAt === jobCreatedAt;+ if (!sameEpoch || !hasDurableReconcileFrame(jobRecord?.finalEvent)) {+ return 'live';+ }+ /** Both reads are scoped to the turn's user and independent, so they run+ * together; a settled row still wins over a persisted anchor. */+ const [settled, anchored] = await Promise.all([responseRowSettled(), anchorExists()]);+ return !settled && anchored ? 'terminal' : 'skip';+}++/**+ * Every id a compaction's finished row can carry: the response id the stream+ * preallocated, and the id the failed-turn path writes its error row under,+ * derived from the anchor exactly as that path derives it. A settled row under+ * either already ends the turn.+ */+function terminalRowIds(anchorMessageId: string, responseMessageId: string): string[] {+ const errorRowId = getPreliminaryResponseMessageId({+ compact: true,+ messageId: anchorMessageId,+ liveResponseMessageId: responseMessageId,+ });+ /** An anchor ending in one underscore derives itself; the failed-turn path+ * then redirects its error row to the response id, so the anchor is never a+ * terminal identity. */+ return errorRowId == null || errorRowId === responseMessageId || errorRowId === anchorMessageId+ ? [responseMessageId]+ : [responseMessageId, errorRowId];+}++/** What the last-subscriber disconnect writes for this turn, if anything. */+export type DisconnectSnapshotPlan =+ | { write: false }+ | {+ write: true;+ content: TMessageContentParts[];+ unfinished: boolean;+ error: boolean;+ };++/**+ * Plans the last-subscriber disconnect snapshot through the caller's message+ * reader: the mode, the marked content, and the row envelope. The caller only+ * builds and writes the row, inside its own tenant context (the reads here+ * run in it too). A live snapshot with nothing persistable is skipped; a+ * promoted terminal snapshot synthesizes its outcome even from empty content.+ */+export async function planDisconnectSnapshot(+ isCompaction: boolean,+ {+ content,+ jobRecord,+ jobCreatedAt,+ userId,+ conversationId,+ anchorMessageId,+ responseMessageId,+ liveSaved = false,+ getMessages,+ onReadError,+ }: {+ /** The streamed content, already filtered to persistable parts. */+ content: TMessageContentParts[];+ jobRecord: Parameters<typeof resolveDisconnectSnapshotMode>[1];+ jobCreatedAt: number | undefined;+ userId: string;+ conversationId: string;+ anchorMessageId: string;+ responseMessageId: string;+ /** This process already wrote a live snapshot of the turn: only a terminal+ * promotion writes again. */+ liveSaved?: boolean;+ getMessages: (+ filter: { user: string; messageId: string; conversationId: string },+ projection?: string,+ ) => Promise<ReadableMessageRow[]>;+ /** Receives a row read that failed while planning. */+ onReadError?: (error: unknown) => void;+ },+): Promise<DisconnectSnapshotPlan> {+ /** A failed row read proves nothing either way: it is reported, nothing is+ * written, and the next disconnect plans again. */+ let mode: DisconnectSnapshotMode;+ try {+ mode = await resolveDisconnectSnapshotMode(isCompaction, jobRecord, jobCreatedAt, {+ anchorExists: async () =>+ (await getMessages({ user: userId, messageId: anchorMessageId, conversationId }, '_id'))+ .length > 0,+ responseRowSettled: async () => {+ const rows = await Promise.all(+ terminalRowIds(anchorMessageId, responseMessageId).map((messageId) =>+ getMessages({ user: userId, messageId, conversationId }, '_id unfinished'),+ ),+ );+ return rows.some(([row]) => row != null && row.unfinished !== true);+ },+ });+ } catch (error) {+ onReadError?.(error);+ return { write: false };+ }+ if (mode === 'skip' || (mode === 'live' && (liveSaved || content.length === 0))) {+ return { write: false };+ }+ const marked = markAbortedCompactionContent(content, isCompaction, {+ synthesizeFailure: mode === 'terminal',+ });+ if (marked.length === 0) {+ return { write: false };+ }+ return {+ write: true,+ content: marked,+ ...(mode === 'terminal'+ ? resolveReconciledSnapshotEnvelope(jobRecord?.status)+ : { unfinished: true, error: false }),+ };+}++/**+ * Stamps a promoted disconnect snapshot on its conversation once it is+ * written: it is the turn's terminal row, so another device must learn the+ * turn ended exactly as it would from any persisted terminal turn. A live+ * snapshot is a fallback, not a reply, and stays unannounced. Never throws.+ */+export async function announceDisconnectSnapshot(+ deps: ErrorTurnStampStore,+ plan: DisconnectSnapshotPlan,+ announcement: ErrorTurnAnnouncement,+): Promise<void> {+ if (!plan.write || plan.unfinished) {+ return;+ }+ await announceErrorTurn(deps, announcement);+}++/**+ * Runs a turn's disconnect snapshots one at a time and plans the snapshot once+ * more when the turn settles, before its terminal cleanup releases the content+ * and resume state the plan reads. Once every subscriber has left, no further+ * disconnect arrives, so a terminal write that failed afterwards would leave a+ * compaction's live snapshot (or nothing, when it left before any content)+ * without its terminal outcome. Serializing keeps an earlier live write from+ * landing over the promoted row.+ */+export function createDisconnectSnapshotQueue({+ isCompaction,+ persist,+ readContent,+ onSettleError,+}: {+ isCompaction: boolean;+ /** Plans and writes one snapshot of the turn's content. */+ persist: (content: TMessageContentParts[]) => Promise<void>;+ /** The turn's stored content once it settled. */+ readContent: () => Promise<TMessageContentParts[]>;+ onSettleError?: (error: unknown) => void;+}): {+ onDisconnect: (content: TMessageContentParts[]) => Promise<void>;+ settle: () => Promise<void>;+} {+ let pending: Promise<void> = Promise.resolve();+ let disconnected = false;+ const enqueue = (write: () => Promise<void>): Promise<void> => {+ const next = pending.then(write);+ pending = next.catch(() => undefined);+ return next;+ };+ return {+ onDisconnect: (content) => {+ disconnected = true;+ return enqueue(() => persist(content));+ },+ settle: async () => {+ if (!isCompaction || !disconnected) {+ return;+ }+ try {+ await enqueue(async () => persist(await readContent()));+ } catch (error) {+ onSettleError?.(error);+ }+ },+ };}/** How the abort route persists a stopped turn's prerequisite rows. */diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 56de9a96ea..1864758b48 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -4578,7 +4578,12 @@ class GenerationJobManagerClass {streamId: string,error?: string,expectedCreatedAt?: number,- options: { beforeErrorPublication?: () => Promise<void> } = {},+ options: {+ beforeErrorPublication?: () => Promise<void>;+ /** Runs once the final event is stored and before the job's content and+ * resume state are released; it must not throw. */+ beforeTerminalCleanup?: () => Promise<void>;+ } = {},): Promise<boolean> {const beforeErrorPublication = error ? options.beforeErrorPublication : undefined;const claim = await this.claimTerminalJob(@@ -4627,6 +4632,7 @@ class GenerationJobManagerClass {}}+ await options.beforeTerminalCleanup?.();await this.finishTerminalJob(claim);return true;}diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.tsindex a595f76974..5ccb8ae587 100644--- a/packages/api/src/stream/__tests__/startup.spec.ts+++ b/packages/api/src/stream/__tests__/startup.spec.ts@@ -3074,6 +3074,40 @@ describe('GenerationJobManager startup telemetry', () => {await manager.destroy();});+ it('runs the terminal cleanup hook after the reconciliation is stored and before cleanup', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ manager.initialize();+ const streamId = 'stream-error-cleanup-hook';+ const job = await manager.createJob(streamId, 'user-1');+ const finishTerminalJob = jest.spyOn(manager, 'finishTerminalJob');+ let seenFinalEvent: string | undefined;+ const beforeTerminalCleanup = jest.fn(async () => {+ expect(finishTerminalJob).not.toHaveBeenCalled();+ seenFinalEvent = (await jobStore.getJob(streamId))?.finalEvent;+ });++ await expect(+ manager.completeJob(streamId, 'initialization failed', job.createdAt, {+ beforeErrorPublication: async () => {+ throw new Error('message store unavailable');+ },+ beforeTerminalCleanup,+ }),+ ).resolves.toBe(true);++ expect(beforeTerminalCleanup).toHaveBeenCalledTimes(1);+ expect(JSON.parse(seenFinalEvent ?? '{}')).toMatchObject({ reconcile: true });+ expect(finishTerminalJob).toHaveBeenCalledTimes(1);+ await manager.destroy();+ });+it('atomically terminalizes a paused job when post-HITL persistence fails', async () => {const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });const manager = new GenerationJobManagerClass();