Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛟 fix: Promote Reconciled Compaction Snapshots to the Terminal Row (#16553)

Marco Beretta0bcb0aaec1d0
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
index 6411c5c831..b582af6379 100644
--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
@@ -17,6 +17,7 @@ const mockTenantStorageRun = jest.fn(async (context, callback) => {
const mockSaveMessage = jest.fn();
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
+const mockStampConvoLastResponse = jest.fn();
const mockIsAgentTriggerPrincipalActive = jest.fn();
const mockFilterPersistableAbortContent = jest.fn((content) => content);
const mockCheckAndIncrementPendingRequest = jest.fn();
@@ -58,10 +59,12 @@ jest.mock('@librechat/api', () => ({
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
applyForcedTemporaryRequest: jest.fn(),
resolveResumableRetention: jest.requireActual('@librechat/api').resolveResumableRetention,
- markAbortedCompactionContent: (...args) =>
- jest.requireActual('@librechat/api').markAbortedCompactionContent(...args),
- resolveDisconnectSnapshotMode: (...args) =>
- jest.requireActual('@librechat/api').resolveDisconnectSnapshotMode(...args),
+ planDisconnectSnapshot: (...args) =>
+ jest.requireActual('@librechat/api').planDisconnectSnapshot(...args),
+ createDisconnectSnapshotQueue: (...args) =>
+ jest.requireActual('@librechat/api').createDisconnectSnapshotQueue(...args),
+ announceDisconnectSnapshot: (...args) =>
+ jest.requireActual('@librechat/api').announceDisconnectSnapshot(...args),
settleExistingRowsBeforeErrorTurn: (...args) =>
jest.requireActual('@librechat/api').settleExistingRowsBeforeErrorTurn(...args),
sendEvent: jest.fn(),
@@ -141,12 +144,19 @@ jest.mock('~/models', () => ({
saveMessage: (...args) => mockSaveMessage(...args),
getMessages: (...args) => mockGetMessages(...args),
getConvo: (...args) => mockGetConvo(...args),
+ stampConvoLastResponse: (...args) => mockStampConvoLastResponse(...args),
isAgentTriggerPrincipalActive: (...args) => mockIsAgentTriggerPrincipalActive(...args),
}));
const AgentController = require('../request');
describe('ResumableAgentController tenant context', () => {
+ /** The anchor is persisted and no response or error row exists yet. */
+ const anchorOnly = () =>
+ mockGetMessages.mockImplementation(async (filter) =>
+ filter.messageId === 'user-message' ? [{ _id: 'existing-anchor' }] : [],
+ );
+
beforeEach(() => {
jest.clearAllMocks();
activeTenantContext = undefined;
@@ -177,6 +187,7 @@ describe('ResumableAgentController tenant context', () => {
fading: { v: 1, budgetTokens: 50_000, masked: true },
};
+ let lastAllSubscribersLeftHandler;
const firePartialDisconnect = async (
user,
jobRecord = { createdAt: 1000, contextMeta: partialContextMeta },
@@ -204,6 +215,7 @@ describe('ResumableAgentController tenant context', () => {
on: jest.fn((event, handler) => {
if (event === 'allSubscribersLeft') {
allSubscribersLeftHandler = handler;
+ lastAllSubscribersLeftHandler = handler;
}
}),
},
@@ -253,6 +265,7 @@ describe('ResumableAgentController tenant context', () => {
it('carries the context meta the run published onto the partial response saved on disconnect', async () => {
await firePartialDisconnect({ id: 'user-123', tenantId: 'tenant-a' });
+ expect(mockStampConvoLastResponse).not.toHaveBeenCalled();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.any(Object),
@@ -361,13 +374,17 @@ describe('ResumableAgentController tenant context', () => {
expect(mockSaveMessage).not.toHaveBeenCalled();
});
- /** Ordinary turns keep the pre-change behavior exactly: their snapshot is
- * the fallback row even when the job record has settled, because the
- * terminal row write may still fail. */
- it('still persists an ordinary partial save when the job record has settled', async () => {
+ /** An ordinary turn's record cannot prove its response row landed (a
+ * terminal transition without a write leaves the same record), so its
+ * snapshot stays the fallback row, in its live shape. */
+ it.each([
+ ['a settled record', {}],
+ ['a pending terminal write', { terminalPersistencePending: true }],
+ ['a reconciliation frame', { finalEvent: JSON.stringify({ final: true, reconcile: true }) }],
+ ])('keeps an ordinary partial save as the fallback behind %s', async (_label, record) => {
await firePartialDisconnect(
{ id: 'user-123' },
- { createdAt: 1000, status: 'error' },
+ { createdAt: 1000, status: 'complete', ...record },
{ aggregatedContent: [{ type: 'text', text: 'Partial response' }] },
);
@@ -375,4 +392,209 @@ describe('ResumableAgentController tenant context', () => {
const [, savedMessage] = mockSaveMessage.mock.calls[0];
expect(savedMessage).toMatchObject({ unfinished: true, error: false });
});
+ /** The terminal write failed and settled for a reconciliation frame: the
+ * snapshot is the turn's only row, so it persists with the terminal
+ * outcome (the typed failure for content with no summary) and envelope. */
+ it('promotes a reconciled compaction snapshot to the terminal row', async () => {
+ anchorOnly();
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ {
+ body: { compact: true },
+ aggregatedContent: [{ type: 'think', think: 'Picking what to summarize' }],
+ },
+ );
+
+ expect(mockSaveMessage).toHaveBeenCalledTimes(1);
+ const [, savedMessage] = mockSaveMessage.mock.calls[0];
+ expect(savedMessage).toMatchObject({ unfinished: false, error: true });
+ expect(savedMessage.content).toEqual([
+ { type: 'think', think: 'Picking what to summarize' },
+ expect.objectContaining({ type: 'error', initiatedBy: 'user' }),
+ ]);
+ expect(mockStampConvoLastResponse).toHaveBeenCalledWith(
+ 'user-123',
+ 'conversation-123',
+ 'response-message',
+ );
+ });
+ /** A reconciliation frame does not prove the row is missing: an error turn
+ * saved before it is a settled row the promotion must not overwrite. */
+ it('withholds a reconciled snapshot when a settled response row exists', async () => {
+ mockGetMessages.mockImplementation(async (filter) =>
+ filter.messageId === 'response-message'
+ ? [{ _id: 'error-row', unfinished: false }]
+ : [{ _id: 'existing-anchor' }],
+ );
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ {
+ body: { compact: true },
+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],
+ },
+ );
+
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ });
+ it('promotes a compaction snapshot on a later disconnect after a live save', async () => {
+ mockGetMessages.mockImplementation(async (filter) =>
+ filter.messageId === 'user-message' ? [{ _id: 'existing-anchor' }] : [],
+ );
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ { createdAt: 1000, status: 'running' },
+ { body: { compact: true } },
+ );
+ expect(mockSaveMessage).toHaveBeenCalledTimes(1);
+ expect(mockSaveMessage.mock.calls[0][1]).toMatchObject({ unfinished: true });
+
+ mockGenerationJobManager.getJobStore.mockReturnValue({
+ getJob: jest.fn().mockResolvedValue({
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ }),
+ });
+ mockGetMessages.mockImplementation(async (filter) =>
+ filter.messageId === 'user-message'
+ ? [{ _id: 'existing-anchor' }]
+ : [{ _id: 'live-row', unfinished: true }],
+ );
+ await lastAllSubscribersLeftHandler([{ type: 'text', text: 'Partial response' }]);
+
+ expect(mockSaveMessage).toHaveBeenCalledTimes(2);
+ expect(mockSaveMessage.mock.calls[1][1]).toMatchObject({ unfinished: false, error: true });
+ });
+
+ it('logs a failed row read while planning the snapshot instead of rejecting the listener', async () => {
+ const readError = new Error('read failed');
+ mockGetMessages.mockImplementation(async (filter) => {
+ if (filter.messageId === 'user-message') {
+ return [{ _id: 'existing-anchor' }];
+ }
+ throw readError;
+ });
+ await expect(
+ firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ {
+ body: { compact: true },
+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],
+ },
+ ),
+ ).resolves.not.toThrow();
+
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ expect(mockLogger.error).toHaveBeenCalledWith(
+ '[ResumableAgentController] Error planning partial response:',
+ readError,
+ );
+ });
+ it('withholds a reconciled snapshot when the error turn saved its anchor-derived row', async () => {
+ mockGetMessages.mockImplementation(
+ async (filter) =>
+ ({
+ 'user-message': [{ _id: 'existing-anchor' }],
+ 'user-message_': [{ _id: 'error-row', unfinished: false }],
+ })[filter.messageId] ?? [],
+ );
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ {
+ body: { compact: true },
+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],
+ },
+ );
+
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ });
+
+ /** Without a persisted anchor the promotion is withheld: the fallback must
+ * not recreate the orphan the absent-anchor abort suppressed. */
+ it('withholds a reconciled snapshot whose anchor was never persisted', async () => {
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ {
+ body: { compact: true },
+ aggregatedContent: [{ type: 'text', text: 'Partial response' }],
+ },
+ );
+
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ });
+
+ /** An empty snapshot on a reconciled compaction still records the turn:
+ * the terminal synthesis appends the typed failure from nothing. */
+ it('synthesizes the terminal outcome for an empty reconciled compaction snapshot', async () => {
+ anchorOnly();
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ { body: { compact: true }, aggregatedContent: [] },
+ );
+
+ expect(mockSaveMessage).toHaveBeenCalledTimes(1);
+ const [, savedMessage] = mockSaveMessage.mock.calls[0];
+ expect(savedMessage).toMatchObject({ unfinished: false, error: true });
+ expect(savedMessage.content).toEqual([
+ expect.objectContaining({ type: 'error', initiatedBy: 'user' }),
+ ]);
+ });
+
+ /** A completed claim's promoted snapshot settles as a finished row, not an
+ * error. */
+ it('settles a completed reconciled snapshot as a finished row', async () => {
+ anchorOnly();
+ await firePartialDisconnect(
+ { id: 'user-123' },
+ {
+ createdAt: 1000,
+ status: 'complete',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ {
+ body: { compact: true },
+ aggregatedContent: [
+ {
+ type: 'summary',
+ content: [{ type: 'text', text: 'A finished checkpoint.' }],
+ boundary: { messageId: 'm', contentIndex: 0 },
+ },
+ ],
+ },
+ );
+
+ expect(mockSaveMessage).toHaveBeenCalledTimes(1);
+ const [, savedMessage] = mockSaveMessage.mock.calls[0];
+ expect(savedMessage).toMatchObject({ unfinished: false, error: false });
+ });
});
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index 0cd2f11aee..f3993086e3 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -273,10 +273,12 @@ jest.mock('@librechat/api', () => ({
jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
applyForcedTemporaryRequest: jest.fn(),
resolveResumableRetention: jest.requireActual('@librechat/api').resolveResumableRetention,
- markAbortedCompactionContent: (...args) =>
- jest.requireActual('@librechat/api').markAbortedCompactionContent(...args),
- resolveDisconnectSnapshotMode: (...args) =>
- jest.requireActual('@librechat/api').resolveDisconnectSnapshotMode(...args),
+ planDisconnectSnapshot: (...args) =>
+ jest.requireActual('@librechat/api').planDisconnectSnapshot(...args),
+ createDisconnectSnapshotQueue: (...args) =>
+ jest.requireActual('@librechat/api').createDisconnectSnapshotQueue(...args),
+ announceDisconnectSnapshot: (...args) =>
+ jest.requireActual('@librechat/api').announceDisconnectSnapshot(...args),
settleExistingRowsBeforeErrorTurn: (...args) =>
jest.requireActual('@librechat/api').settleExistingRowsBeforeErrorTurn(...args),
sendEvent: jest.fn(),
@@ -5191,6 +5193,138 @@ describe('ResumableAgentController resume metadata', () => {
},
);
+ it('promotes a live compaction snapshot once its terminal write settles for reconciliation', async () => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: '',
+ };
+ const textPart = { type: 'text', text: 'Partial summary' };
+ let jobRecord = { createdAt: 1000, status: 'running' };
+ let allSubscribersLeftHandler;
+ mockGenerationJobManager.createJob.mockResolvedValue({
+ createdAt: 1000,
+ metadata: { providerExecutionId: 'provider-segment-1', providerDrained: true },
+ readyPromise: Promise.resolve(),
+ abortController: new AbortController(),
+ emitter: {
+ on: jest.fn((event, handler) => {
+ if (event === 'allSubscribersLeft') {
+ allSubscribersLeftHandler = handler;
+ }
+ }),
+ },
+ });
+ mockGenerationJobManager.getJobStore.mockReturnValue({
+ getJob: jest.fn(async () => jobRecord),
+ getContentParts: jest.fn(async () => ({ content: [textPart] })),
+ });
+ mockGenerationJobManager.getResumeState.mockResolvedValue({
+ conversationId: 'conversation-123',
+ responseMessageId: 'response-msg',
+ userMessage: { messageId: userMessage.messageId },
+ });
+ mockGenerationJobManager.publishTerminalClaim.mockImplementation(async (_claim, finalEvent) => {
+ if (finalEvent == null) {
+ jobRecord = {
+ createdAt: 1000,
+ status: 'complete',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ };
+ }
+ return { finalEvent, persistenceFailed: finalEvent == null };
+ });
+ mockGetMessages.mockImplementation(async ({ messageId }) => {
+ if (messageId === userMessage.messageId) {
+ return [{ _id: 'anchor' }];
+ }
+ return messageId === 'response-msg' ? [{ _id: 'live-row', unfinished: true }] : [];
+ });
+ let signalPromoted;
+ const promoted = new Promise((resolve) => {
+ signalPromoted = resolve;
+ });
+ let signalFinished;
+ const finished = new Promise((resolve) => {
+ signalFinished = resolve;
+ });
+ mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());
+ mockSaveMessage.mockImplementation(async (_reqCtx, message) => {
+ if (message?.messageId === userMessage.messageId) {
+ return {};
+ }
+ if (message?.unfinished === false && message?.error === false) {
+ signalPromoted(message);
+ return {};
+ }
+ /** The live snapshot lands; the completed response write does not. */
+ return message?.unfinished === true ? {} : undefined;
+ });
+ const client = {
+ options: {},
+ savedMessageIds: new Set(['response-msg']),
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ await allSubscribersLeftHandler([textPart]);
+ return {
+ messageId: 'response-msg',
+ parentMessageId: userMessage.messageId,
+ conversationId: 'conversation-123',
+ content: [textPart],
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123', title: 'Existing' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: '',
+ compact: true,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.messageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ const [promotedRow] = await Promise.all([promoted, finished]);
+
+ const snapshotWrites = mockSaveMessage.mock.calls
+ .map(([, message]) => message)
+ .filter((message) => message.messageId === 'response-msg' && message.unfinished != null);
+ expect(snapshotWrites[0]).toMatchObject({ unfinished: true, error: false });
+ expect(promotedRow).toMatchObject({
+ messageId: 'response-msg',
+ parentMessageId: userMessage.messageId,
+ unfinished: false,
+ error: false,
+ });
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(
+ expect.anything(),
+ null,
+ );
+ /** Terminal cleanup releases the content and resume state the plan reads. */
+ const promotedOrder =
+ mockSaveMessage.mock.invocationCallOrder[
+ mockSaveMessage.mock.calls.findIndex(([, message]) => message === promotedRow)
+ ];
+ expect(promotedOrder).toBeLessThan(
+ mockGenerationJobManager.finishTerminalJob.mock.invocationCallOrder[0],
+ );
+ });
+
it('suppresses the completed response write when Stop already won terminal ownership', async () => {
const userMessage = {
messageId: 'user-msg',
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index d43a1f9849..8750242ffd 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -63,10 +63,11 @@ const {
getUserFacingGenerationError,
stampPreliminaryPrivateTextMessage,
announceReply,
+ announceDisconnectSnapshot,
announceErrorTurn,
settleExistingRowsBeforeErrorTurn,
- resolveDisconnectSnapshotMode,
- markAbortedCompactionContent,
+ planDisconnectSnapshot,
+ createDisconnectSnapshotQueue,
isTemporaryTurn,
getPreliminaryResponseMessageId,
} = require('@librechat/api');
@@ -1903,21 +1904,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
* actual response messageId isn't available yet. The final response save will
* overwrite this with the complete response using the same messageId pattern.
*/
- job.emitter.on('allSubscribersLeft', async (aggregatedContent) => {
- if (partialResponseSaved || !aggregatedContent || aggregatedContent.length === 0) {
- return;
- }
-
- /** The run is still live here: mark what streamed, but leave the outcome
- * to whichever path settles the turn (the terminal abort synthesizes
- * the typed failure a stopped compaction with no summary needs). */
- const persistableContent = markAbortedCompactionContent(
- filterPersistableAbortContent(aggregatedContent),
- isCompaction,
- { synthesizeFailure: false },
- );
- if (persistableContent.length === 0) {
- logger.debug('[ResumableAgentController] No persistable content to save partial response');
+ const persistDisconnectSnapshot = async (aggregatedContent) => {
+ /** Empty content is rejected only after the snapshot mode is known: a
+ * reconciled compaction synthesizes its terminal outcome from nothing,
+ * while a live run has nothing persistable. Whether an earlier live
+ * save still allows a write is part of that plan. */
+ if (!aggregatedContent) {
return;
}
@@ -1930,8 +1922,34 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
return;
}
- partialResponseSaved = true;
+ /** Whether and how this snapshot persists is planned in @librechat/api,
+ * inside the request's tenant context like the write below. */
+ const responseMessageId =
+ resumeState.responseMessageId || `${resumeState.userMessage.messageId}_`;
const responseConversationId = resumeState.conversationId || conversationId;
+ const planSnapshot = () =>
+ planDisconnectSnapshot(isCompaction, {
+ content: filterPersistableAbortContent(aggregatedContent),
+ jobRecord,
+ jobCreatedAt,
+ userId,
+ conversationId: responseConversationId,
+ anchorMessageId: resumeState.userMessage.messageId,
+ responseMessageId,
+ liveSaved: partialResponseSaved,
+ getMessages,
+ onReadError: (error) =>
+ logger.error('[ResumableAgentController] Error planning partial response:', error),
+ });
+ const snapshot = tenantId
+ ? await tenantStorage.run({ tenantId, userId }, planSnapshot)
+ : await planSnapshot();
+ if (!snapshot.write) {
+ logger.debug('[ResumableAgentController] No partial response to save on disconnect');
+ return;
+ }
+
+ partialResponseSaved = true;
/** The run publishes its calibration and fading tiers onto the job; a
* partial response saved on disconnect must carry them like the Stop and
* pause paths do, or a turn continued from it re-derives its provider
@@ -1939,22 +1957,16 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
* record is the source, since the client-facing resume snapshot never
* carries server-private state. */
const contextMeta = jobRecord?.createdAt === jobCreatedAt ? jobRecord.contextMeta : undefined;
- if (resolveDisconnectSnapshotMode(isCompaction, jobRecord, jobCreatedAt) === 'skip') {
- logger.debug(
- '[ResumableAgentController] Skipping compaction partial save for a settled job',
- );
- return;
- }
try {
const partialMessage = {
- messageId: resumeState.responseMessageId || `${resumeState.userMessage.messageId}_`,
+ messageId: responseMessageId,
conversationId: responseConversationId,
parentMessageId: resumeState.userMessage.messageId,
sender: client?.sender ?? 'AI',
- content: persistableContent,
- unfinished: true,
- error: false,
+ content: snapshot.content,
+ unfinished: snapshot.unfinished,
+ error: snapshot.error,
isCreatedByUser: false,
user: userId,
endpoint: endpointOption.endpoint,
@@ -1967,14 +1979,15 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
partialMessage.agent_id = req.body.agent_id;
}
+ const isTemporary =
+ req?._agentEventBindingRetention?.isTemporary ??
+ req?.resolvedConversation?.isTemporary ??
+ req?.body?.isTemporary;
const savePartialMessage = () =>
saveMessage(
{
userId,
- isTemporary:
- req?._agentEventBindingRetention?.isTemporary ??
- req?.resolvedConversation?.isTemporary ??
- req?.body?.isTemporary,
+ isTemporary,
expiredAt:
req?._agentEventBindingRetention?.expiredAt ?? req?.resolvedConversation?.expiredAt,
interfaceConfig: req?.config?.interfaceConfig,
@@ -1992,15 +2005,40 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
throw new Error('Partial response could not be persisted after disconnect');
}
+ const announcePromotion = () =>
+ announceDisconnectSnapshot({ stampConvoLastResponse }, snapshot, {
+ userId,
+ conversationId: responseConversationId,
+ messageId: responseMessageId,
+ isTemporary,
+ context: 'ResumableAgentController - promoted disconnect snapshot',
+ });
+ await (tenantId
+ ? tenantStorage.run({ tenantId, userId }, announcePromotion)
+ : announcePromotion());
+
logger.debug(
- `[ResumableAgentController] Saved partial response for ${streamId}, content parts: ${persistableContent.length}`,
+ `[ResumableAgentController] Saved partial response for ${streamId}, content parts: ${snapshot.content.length}`,
);
} catch (error) {
logger.error('[ResumableAgentController] Error saving partial response:', error);
// Reset flag so we can try again if subscribers reconnect and leave again
partialResponseSaved = false;
}
+ };
+ const disconnectSnapshots = createDisconnectSnapshotQueue({
+ isCompaction,
+ persist: persistDisconnectSnapshot,
+ readContent: async () =>
+ (await GenerationJobManager.getJobStore().getContentParts(streamId, jobCreatedAt))
+ ?.content ?? [],
+ onSettleError: (error) =>
+ logger.error(
+ '[ResumableAgentController] Error settling the saved partial response:',
+ error,
+ ),
});
+ job.emitter.on('allSubscribersLeft', disconnectSnapshots.onDisconnect);
/** @type {{ client: TAgentClient; userMCPAuthMap?: Record<string, Record<string, string>> }} */
const result = await initializeClient({
@@ -2204,6 +2242,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
return;
}
try {
+ await disconnectSnapshots.settle();
await GenerationJobManager.finishTerminalJob(terminalClaim);
} finally {
terminalClaimFinished = true;
@@ -3408,6 +3447,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
initialAgentId: verifiedInitialAgentId,
}),
),
+ beforeTerminalCleanup: disconnectSnapshots.settle,
})) === true;
/** A true completion means this owner won the terminal CAS and
* the beforeErrorPublication barrier above finished. Only that
diff --git a/e2e/specs/mock/scenarios/compaction-snapshot-modes.spec.ts b/e2e/specs/mock/scenarios/compaction-snapshot-modes.spec.ts
new file mode 100644
index 0000000000..2ae0745ef2
--- /dev/null
+++ b/e2e/specs/mock/scenarios/compaction-snapshot-modes.spec.ts
@@ -0,0 +1,106 @@
+import { expect, test } from '@playwright/test';
+import { randomUUID } from 'node:crypto';
+import { getE2EUser } from '../../../setup/user';
+import { deleteConversations, deleteMessagesByConversation, seedMessages, withMongo } from '../db';
+import { messagesView, sendMessageAndWaitForCompletion } from '../helpers';
+
+const userEmail = getE2EUser().email;
+const LABEL_SERVER = `http://127.0.0.1:${process.env.E2E_LABEL_PORT || '8889'}`;
+
+type Row = Record<string, unknown>;
+
+async function cleanup(conversationId: string) {
+ await deleteMessagesByConversation([conversationId]);
+ await deleteConversations([conversationId]);
+}
+
+function findRow(filter: Row): Promise<Row | null> {
+ return withMongo((db) => db.collection('messages').findOne(filter));
+}
+
+test.describe('compaction snapshot modes', () => {
+ test.afterEach(async ({ request }) => {
+ const response = await request.post(`${LABEL_SERVER}/__e2e/reset`);
+ expect(response.ok()).toBeTruthy();
+ });
+
+ /* The last viewer leaving a live compaction saves its snapshot in the live
+ shape; the run then finishes on its own and its terminal row replaces the
+ snapshot, so the reopened conversation shows a settled compaction. */
+ test('a compaction whose viewer leaves mid-run still settles after it finishes @scenario:abandoned-compaction-settles-after-the-run', async ({
+ page,
+ request,
+ }) => {
+ test.setTimeout(150_000);
+ await page.goto('/c/new');
+ await sendMessageAndWaitForCompletion(page, 'tell me about abandoned compactions');
+ const conversationId = new URL(page.url()).pathname.replace('/c/', '');
+ expect(conversationId).not.toBe('new');
+
+ try {
+ const answer = await withMongo((db) =>
+ db
+ .collection('messages')
+ .findOne({ conversationId, isCreatedByUser: false }, { sort: { createdAt: -1 } }),
+ );
+ expect(answer?.messageId).toBeTruthy();
+
+ const leafId = randomUUID();
+ const leafText = 'Compact this and walk away';
+ await seedMessages(userEmail, conversationId, [
+ {
+ messageId: leafId,
+ parentMessageId: answer?.messageId as string,
+ text: leafText,
+ isCreatedByUser: true,
+ sender: 'User',
+ },
+ ]);
+
+ const behavior = await request.post(`${LABEL_SERVER}/__e2e/behavior`, {
+ data: { mode: 'ok', delayMs: 8_000 },
+ });
+ expect(behavior.ok()).toBeTruthy();
+
+ await page.goto(`/c/${conversationId}`);
+ await expect(messagesView(page).getByText(leafText)).toBeVisible();
+ await page.getByTestId('token-usage').click();
+ await page.getByRole('button', { name: 'Compact context' }).click();
+ await expect(page.getByTestId('stop-generation-button')).toBeVisible({ timeout: 20_000 });
+
+ /* Closing the only viewer drops every subscriber while the summary runs. */
+ const context = page.context();
+ await page.close();
+
+ let compaction: Row | null = null;
+ await expect
+ .poll(
+ async () => {
+ compaction = await findRow({
+ conversationId,
+ parentMessageId: leafId,
+ isCreatedByUser: false,
+ });
+ return compaction != null && compaction.unfinished !== true;
+ },
+ { timeout: 60_000, intervals: [2_000] },
+ )
+ .toBeTruthy();
+ expect((compaction as Row | null)?.error).not.toBe(true);
+
+ const leaf = await findRow({ conversationId, messageId: leafId });
+ expect(leaf?.isCreatedByUser).toBe(true);
+ expect(leaf?.text).toBe(leafText);
+
+ const viewer = await context.newPage();
+ await viewer.goto(`/c/${conversationId}`);
+ await expect(messagesView(viewer).getByText(leafText)).toBeVisible({ timeout: 20_000 });
+ const row = viewer.locator(`[id="${(compaction as Row | null)?.messageId as string}"]`);
+ await expect(row).toBeVisible({ timeout: 20_000 });
+ await expect(row.getByText('Summarizing...')).toHaveCount(0);
+ await expect(row.getByText('Could not compact the context', { exact: false })).toHaveCount(0);
+ } finally {
+ await cleanup(conversationId);
+ }
+ });
+});
diff --git a/packages/api/src/agents/compaction.spec.ts b/packages/api/src/agents/compaction.spec.ts
index 2e58d808f9..9a75153f4f 100644
--- a/packages/api/src/agents/compaction.spec.ts
+++ b/packages/api/src/agents/compaction.spec.ts
@@ -20,7 +20,11 @@ import {
persistFinalizedCompactionTurn,
isSettledJobRecord,
resolveDisconnectSnapshotMode,
+ resolveReconciledSnapshotEnvelope,
planAbortedTurnPersistence,
+ planDisconnectSnapshot,
+ createDisconnectSnapshotQueue,
+ announceDisconnectSnapshot,
resolveAbortedTurnPersistence,
resolveAbortedTurnAnchorDecision,
settleExistingRowsBeforeErrorTurn,
@@ -31,6 +35,7 @@ import {
stripUnusableSummaryParts,
getLatestEventActorSummary,
} from './compaction';
+import { MAX_RESPONSE_MESSAGE_ID_LENGTH } from '~/utils/message';
const index = [
{
@@ -1434,6 +1439,25 @@ describe('isSettledJobRecord', () => {
).toBe(false);
});
+ /** A failed terminal write clears the marker while publishing a
+ * reconciliation frame: no row was persisted, so the streamed snapshot
+ * stays the turn's only fallback. */
+ it('treats a record whose durable final event is a reconciliation frame as unsettled', () => {
+ const reconciled = {
+ createdAt: 1000,
+ status: 'aborted',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ };
+ const settled = {
+ createdAt: 1000,
+ status: 'aborted',
+ finalEvent: JSON.stringify({ final: true }),
+ };
+
+ expect(isSettledJobRecord(reconciled)).toBe(false);
+ expect(isSettledJobRecord(settled)).toBe(true);
+ });
+
it('leaves live and missing records unsettled', () => {
expect(isSettledJobRecord({ createdAt: 1000, status: 'running' })).toBe(false);
expect(isSettledJobRecord({ createdAt: 1000, status: 'requires_action' })).toBe(false);
@@ -1451,26 +1475,442 @@ describe('isSettledJobRecord', () => {
describe('resolveDisconnectSnapshotMode', () => {
it.each(['complete', 'error', 'aborted'])(
'withholds the snapshot of a %s compaction job',
- (status) => {
- expect(resolveDisconnectSnapshotMode(true, { createdAt: 1000, status }, 1000)).toBe('skip');
+ async (status) => {
+ await expect(
+ resolveDisconnectSnapshotMode(true, { createdAt: 1000, status }, 1000),
+ ).resolves.toBe('skip');
},
);
- it('writes the snapshot for a live, missing, or other-epoch record', () => {
- expect(resolveDisconnectSnapshotMode(true, { createdAt: 1000, status: 'running' }, 1000)).toBe(
- 'live',
+ it('writes the snapshot for a live, missing, or other-epoch record', async () => {
+ await expect(
+ resolveDisconnectSnapshotMode(true, { createdAt: 1000, status: 'running' }, 1000),
+ ).resolves.toBe('live');
+ await expect(resolveDisconnectSnapshotMode(true, null, 1000)).resolves.toBe('live');
+ await expect(
+ resolveDisconnectSnapshotMode(true, { createdAt: 2000, status: 'error' }, 1000),
+ ).resolves.toBe('live');
+ });
+
+ /** An ordinary turn's record cannot prove its response row landed, so its
+ * snapshot stays the fallback whatever the record says. */
+ it('writes the snapshot of a settled ordinary turn', async () => {
+ await expect(
+ resolveDisconnectSnapshotMode(false, { createdAt: 1000, status: 'complete' }, 1000),
+ ).resolves.toBe('live');
+ });
+
+ /** Only a compaction is promoted: an ordinary turn whose terminal write
+ * reconciled keeps its live snapshot as the fallback row. */
+ it('keeps a reconciled ordinary turn on its live snapshot', async () => {
+ const anchorExists = jest.fn(async () => true);
+ await expect(
+ resolveDisconnectSnapshotMode(
+ false,
+ {
+ createdAt: 1000,
+ status: 'complete',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ },
+ 1000,
+ { anchorExists },
+ ),
+ ).resolves.toBe('live');
+ expect(anchorExists).not.toHaveBeenCalled();
+ });
+
+ const reconciled = () => ({
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ });
+
+ /** The terminal write failed and settled for a reconciliation frame: the
+ * snapshot is promoted to the turn's terminal row, because no other row
+ * will ever be persisted for it. */
+ it('promotes a reconciled compaction snapshot to the terminal row', async () => {
+ await expect(resolveDisconnectSnapshotMode(true, reconciled(), 1000)).resolves.toBe('terminal');
+ });
+
+ /** A reconciliation frame can follow a saved error turn: a settled
+ * response row withholds the promotion instead of being overwritten. */
+ it('withholds a reconciled snapshot when the response row is settled', async () => {
+ await expect(
+ resolveDisconnectSnapshotMode(true, reconciled(), 1000, {
+ responseRowSettled: async () => true,
+ }),
+ ).resolves.toBe('skip');
+ });
+
+ it('keeps the live snapshot for another epoch reconciliation frame', async () => {
+ const anchorExists = jest.fn(async () => true);
+ await expect(
+ resolveDisconnectSnapshotMode(true, { ...reconciled(), createdAt: 2000 }, 1000, {
+ anchorExists,
+ }),
+ ).resolves.toBe('live');
+ expect(anchorExists).not.toHaveBeenCalled();
+ });
+
+ /** The promotion must not recreate the orphan an absent-anchor abort
+ * deliberately withheld: without a persisted anchor there is nothing to
+ * hang the terminal row on. */
+ it('withholds a reconciled snapshot whose anchor was never persisted', async () => {
+ await expect(
+ resolveDisconnectSnapshotMode(true, reconciled(), 1000, {
+ anchorExists: async () => false,
+ }),
+ ).resolves.toBe('skip');
+ });
+});
+
+describe('resolveReconciledSnapshotEnvelope', () => {
+ /** The abort route writes a stopped compaction settled, so its
+ * reconciled fallback must not reopen it as a live row. */
+ it('settles an aborted claim like the abort route writes a stopped compaction', () => {
+ expect(resolveReconciledSnapshotEnvelope('aborted')).toEqual({
+ unfinished: false,
+ error: false,
+ });
+ });
+
+ it('settles a completed claim as a finished row', () => {
+ expect(resolveReconciledSnapshotEnvelope('complete')).toEqual({
+ unfinished: false,
+ error: false,
+ });
+ });
+
+ it('settles everything else with the error envelope', () => {
+ expect(resolveReconciledSnapshotEnvelope('error')).toEqual({
+ unfinished: false,
+ error: true,
+ });
+ expect(resolveReconciledSnapshotEnvelope(undefined)).toEqual({
+ unfinished: false,
+ error: true,
+ });
+ });
+});
+
+describe('planDisconnectSnapshot', () => {
+ const reconciled = {
+ createdAt: 1000,
+ status: 'error',
+ finalEvent: JSON.stringify({ final: true, reconcile: true }),
+ };
+ const plan = (
+ isCompaction: boolean,
+ jobRecord: Parameters<typeof planDisconnectSnapshot>[1]['jobRecord'],
+ content: TMessageContentParts[],
+ rows: Record<string, Array<{ messageId: string; unfinished?: boolean }>>,
+ ) => {
+ const getMessages = jest.fn(
+ async (filter: { messageId: string }) => rows[filter.messageId] ?? [],
);
- expect(resolveDisconnectSnapshotMode(true, null, 1000)).toBe('live');
- expect(resolveDisconnectSnapshotMode(true, { createdAt: 2000, status: 'error' }, 1000)).toBe(
- 'live',
+ return {
+ getMessages,
+ result: planDisconnectSnapshot(isCompaction, {
+ content,
+ jobRecord,
+ jobCreatedAt: 1000,
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ anchorMessageId: 'anchor',
+ responseMessageId: 'response',
+ getMessages,
+ }),
+ };
+ };
+ const text = { type: ContentTypes.TEXT, text: 'Partial' } as TMessageContentParts;
+
+ it('writes a live snapshot in the live shape without reading any row', async () => {
+ const { result, getMessages } = plan(
+ false,
+ { createdAt: 1000, status: 'complete' },
+ [text],
+ {},
);
+
+ await expect(result).resolves.toEqual({
+ write: true,
+ content: [text],
+ unfinished: true,
+ error: false,
+ });
+ expect(getMessages).not.toHaveBeenCalled();
});
- /** An ordinary turn's snapshot is the fallback row its terminal write may
- * still need, so it is written whatever the record says. */
- it('writes the snapshot of a settled ordinary turn', () => {
- expect(
- resolveDisconnectSnapshotMode(false, { createdAt: 1000, status: 'complete' }, 1000),
- ).toBe('live');
+ it('writes nothing for an empty live snapshot', async () => {
+ const { result } = plan(true, { createdAt: 1000, status: 'running' }, [], {});
+
+ await expect(result).resolves.toEqual({ write: false });
+ });
+
+ it('promotes a reconciled compaction under its anchor with the terminal envelope', async () => {
+ const { result } = plan(true, reconciled, [], { anchor: [{ messageId: 'anchor' }] });
+ const planned = await result;
+
+ expect(planned).toMatchObject({ write: true, unfinished: false, error: true });
+ expect(planned.write && planned.content).toEqual([
+ expect.objectContaining({ type: ContentTypes.ERROR, initiatedBy: 'user' }),
+ ]);
+ });
+
+ it('promotes over an earlier live snapshot of the same turn', async () => {
+ const { result } = plan(true, reconciled, [text], {
+ anchor: [{ messageId: 'anchor' }],
+ response: [{ messageId: 'response', unfinished: true }],
+ });
+
+ await expect(result).resolves.toMatchObject({ write: true, unfinished: false });
+ });
+
+ it('withholds the promotion when a settled response row exists', async () => {
+ const { result } = plan(true, reconciled, [text], {
+ anchor: [{ messageId: 'anchor' }],
+ response: [{ messageId: 'response', unfinished: false }],
+ });
+
+ await expect(result).resolves.toEqual({ write: false });
+ });
+
+ it('reports a failed row read and writes nothing', async () => {
+ const readError = new Error('read failed');
+ const onReadError = jest.fn();
+
+ await expect(
+ planDisconnectSnapshot(true, {
+ content: [text],
+ jobRecord: reconciled,
+ jobCreatedAt: 1000,
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ anchorMessageId: 'anchor',
+ responseMessageId: 'response',
+ getMessages: jest.fn(async () => {
+ throw readError;
+ }),
+ onReadError,
+ }),
+ ).resolves.toEqual({ write: false });
+ expect(onReadError).toHaveBeenCalledWith(readError);
+ });
+
+ it('writes a live snapshot once but still promotes after it', async () => {
+ const live = planDisconnectSnapshot(true, {
+ content: [text],
+ jobRecord: { createdAt: 1000, status: 'running' },
+ jobCreatedAt: 1000,
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ anchorMessageId: 'anchor',
+ responseMessageId: 'response',
+ liveSaved: true,
+ getMessages: jest.fn(async () => []),
+ });
+ await expect(live).resolves.toEqual({ write: false });
+
+ const getMessages = jest.fn(async (filter: { messageId: string }) =>
+ filter.messageId === 'anchor'
+ ? [{ messageId: 'anchor' }]
+ : [{ messageId: filter.messageId, unfinished: true }],
+ );
+ const promoted = planDisconnectSnapshot(true, {
+ content: [text],
+ jobRecord: reconciled,
+ jobCreatedAt: 1000,
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ anchorMessageId: 'anchor',
+ responseMessageId: 'response',
+ liveSaved: true,
+ getMessages,
+ });
+ await expect(promoted).resolves.toMatchObject({ write: true, unfinished: false });
+ });
+
+ it('withholds the promotion when the failed-turn path already saved its error row', async () => {
+ const { result, getMessages } = plan(true, reconciled, [text], {
+ anchor: [{ messageId: 'anchor' }],
+ anchor_: [{ messageId: 'anchor_', unfinished: false }],
+ });
+
+ await expect(result).resolves.toEqual({ write: false });
+ expect(getMessages).toHaveBeenCalledWith(
+ expect.objectContaining({ messageId: 'anchor_' }),
+ '_id unfinished',
+ );
+ });
+
+ it('still promotes when the derived error row is only an unfinished snapshot', async () => {
+ const { result } = plan(true, reconciled, [text], {
+ anchor: [{ messageId: 'anchor' }],
+ anchor_: [{ messageId: 'anchor_', unfinished: true }],
+ });
+
+ await expect(result).resolves.toMatchObject({ write: true, unfinished: false });
+ });
+
+ it('never treats an anchor ending in one underscore as its own terminal row', async () => {
+ const getMessages = jest.fn(async (filter: { messageId: string }) =>
+ filter.messageId === 'anchor_' ? [{ messageId: 'anchor_', unfinished: false }] : [],
+ );
+
+ await expect(
+ planDisconnectSnapshot(true, {
+ content: [text],
+ jobRecord: reconciled,
+ jobCreatedAt: 1000,
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ anchorMessageId: 'anchor_',
+ responseMessageId: 'response',
+ getMessages,
+ }),
+ ).resolves.toMatchObject({ write: true, unfinished: false });
+ expect(getMessages).toHaveBeenCalledWith(
+ expect.objectContaining({ messageId: 'response' }),
+ '_id unfinished',
+ );
+ expect(getMessages).not.toHaveBeenCalledWith(
+ expect.objectContaining({ messageId: 'anchor_' }),
+ '_id unfinished',
+ );
+ });
+
+ it('reads only the response row when a full-length anchor leaves no room for the suffix', async () => {
+ const anchorMessageId = 'a'.repeat(MAX_RESPONSE_MESSAGE_ID_LENGTH);
+ const getMessages = jest.fn(async (filter: { messageId: string }) =>
+ filter.messageId === anchorMessageId ? [{ messageId: anchorMessageId }] : [],
+ );
+
+ await expect(
+ planDisconnectSnapshot(true, {
+ content: [text],
+ jobRecord: reconciled,
+ jobCreatedAt: 1000,
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ anchorMessageId,
+ responseMessageId: 'response',
+ getMessages,
+ }),
+ ).resolves.toMatchObject({ write: true, unfinished: false });
+ expect(getMessages.mock.calls.map(([filter]) => filter.messageId).sort()).toEqual(
+ [anchorMessageId, 'response'].sort(),
+ );
+ });
+});
+
+describe('createDisconnectSnapshotQueue', () => {
+ const text = { type: ContentTypes.TEXT, text: 'Partial' } as TMessageContentParts;
+ const stored = { type: ContentTypes.TEXT, text: 'Stored' } as TMessageContentParts;
+
+ it('plans once more from the stored content when a disconnected compaction settles', async () => {
+ const persist = jest.fn(async () => undefined);
+ const queue = createDisconnectSnapshotQueue({
+ isCompaction: true,
+ persist,
+ readContent: async () => [stored],
+ });
+
+ await queue.onDisconnect([]);
+ await queue.settle();
+
+ expect(persist.mock.calls).toEqual([[[]], [[stored]]]);
+ });
+
+ it('does not plan again when nobody disconnected or the turn is not a compaction', async () => {
+ const persist = jest.fn(async () => undefined);
+ const readContent = jest.fn(async () => [stored]);
+
+ await createDisconnectSnapshotQueue({ isCompaction: true, persist, readContent }).settle();
+ const ordinary = createDisconnectSnapshotQueue({ isCompaction: false, persist, readContent });
+ await ordinary.onDisconnect([text]);
+ await ordinary.settle();
+
+ expect(persist.mock.calls).toEqual([[[text]]]);
+ expect(readContent).not.toHaveBeenCalled();
+ });
+
+ it('waits for an outstanding snapshot write before the settle plan starts', async () => {
+ const order: string[] = [];
+ let finishLive: () => void = () => undefined;
+ const persist = jest.fn(async (content: TMessageContentParts[]) => {
+ const label = content[0] === stored ? 'settle' : 'live';
+ order.push(`${label}:start`);
+ if (label === 'live') {
+ await new Promise<void>((resolve) => {
+ finishLive = resolve;
+ });
+ }
+ order.push(`${label}:end`);
+ });
+ const queue = createDisconnectSnapshotQueue({
+ isCompaction: true,
+ persist,
+ readContent: async () => [stored],
+ });
+
+ const live = queue.onDisconnect([text]);
+ const settled = queue.settle();
+ await Promise.resolve();
+ finishLive();
+ await Promise.all([live, settled]);
+
+ expect(order).toEqual(['live:start', 'live:end', 'settle:start', 'settle:end']);
+ });
+
+ it('reports a failed settle plan and still runs after a failed live write', async () => {
+ const liveError = new Error('live failed');
+ const settleError = new Error('read failed');
+ const onSettleError = jest.fn();
+ const queue = createDisconnectSnapshotQueue({
+ isCompaction: true,
+ persist: jest.fn(async () => {
+ throw liveError;
+ }),
+ readContent: async () => {
+ throw settleError;
+ },
+ onSettleError,
+ });
+
+ await expect(queue.onDisconnect([text])).rejects.toBe(liveError);
+ await expect(queue.settle()).resolves.toBeUndefined();
+ expect(onSettleError).toHaveBeenCalledWith(settleError);
+ });
+});
+
+describe('announceDisconnectSnapshot', () => {
+ const announcement = {
+ userId: 'user-1',
+ conversationId: 'convo-1',
+ messageId: 'response',
+ context: 'test',
+ };
+
+ it('stamps a promoted terminal snapshot on its conversation', async () => {
+ const stampConvoLastResponse = jest.fn(async () => null);
+ await announceDisconnectSnapshot(
+ { stampConvoLastResponse },
+ { write: true, content: [], unfinished: false, error: true },
+ announcement,
+ );
+
+ expect(stampConvoLastResponse).toHaveBeenCalledWith('user-1', 'convo-1', 'response');
+ });
+
+ it('leaves a live or unwritten snapshot unannounced', async () => {
+ const stampConvoLastResponse = jest.fn(async () => null);
+ await announceDisconnectSnapshot(
+ { stampConvoLastResponse },
+ { write: true, content: [], unfinished: true, error: false },
+ announcement,
+ );
+ await announceDisconnectSnapshot({ stampConvoLastResponse }, { write: false }, announcement);
+
+ expect(stampConvoLastResponse).not.toHaveBeenCalled();
});
});
diff --git a/packages/api/src/agents/compaction.ts b/packages/api/src/agents/compaction.ts
index cbf3c5e66f..c4afa57a8a 100644
--- a/packages/api/src/agents/compaction.ts
+++ b/packages/api/src/agents/compaction.ts
@@ -18,7 +18,10 @@ import type {
} from '@librechat/data-schemas';
import type { SummaryContentPart, TMessageContentParts } from 'librechat-data-provider';
import type { IAgentEventActorSummary } from '@librechat/data-schemas';
+import type { ErrorTurnAnnouncement, ErrorTurnStampStore } from '~/conversations/announce';
+import { getPreliminaryResponseMessageId } from '~/utils/message';
import { createAgentEventActorSummary } from './compatibility';
+import { announceErrorTurn } from '~/conversations/announce';
/** Text of a summary content part, in any persisted shape — `content` blocks
* today, a string `content` or a bare `text` on rows written before them.
@@ -277,16 +280,33 @@ export function markAbortedCompactionContent(
return marked;
}
+/** Whether a job record's durable final event is a reconciliation frame: the
+ * conservative substitute published when the terminal row write failed, so
+ * no message row backs the terminal claim. */
+function hasDurableReconcileFrame(finalEvent: unknown): boolean {
+ if (typeof finalEvent !== 'string' || finalEvent.length === 0) {
+ return false;
+ }
+ try {
+ const parsed = JSON.parse(finalEvent) as { reconcile?: unknown } | null;
+ return parsed?.reconcile === true;
+ } catch {
+ return false;
+ }
+}
+
/** Whether a job record has reached a status whose path owns the turn's final
* row (completion, error, or abort): the disconnect snapshot must not be
* written over it, or the settled row reopens as an unfinished response.
- * Only a same-epoch record is trusted. */
+ * Only a same-epoch record is trusted, and only one whose terminal write
+ * actually landed. */
export function isSettledJobRecord(
jobRecord:
| {
createdAt?: number;
status?: string;
terminalPersistencePending?: boolean;
+ finalEvent?: string;
}
| null
| undefined,
@@ -301,6 +321,12 @@ export function isSettledJobRecord(
* that write fails. */
return false;
}
+ if (hasDurableReconcileFrame(jobRecord.finalEvent)) {
+ /** The terminal write failed and a reconciliation frame was published in
+ * its place: nothing was persisted for the turn, so the streamed
+ * snapshot remains its only row. */
+ return false;
+ }
return (
jobRecord.status === 'complete' ||
jobRecord.status === 'error' ||
@@ -308,31 +334,257 @@ export function isSettledJobRecord(
);
}
-/** How the last-subscriber disconnect may persist this turn's snapshot. */
+/** How a disconnect may persist this turn's snapshot. */
export type DisconnectSnapshotMode =
- /** The run is still live: the snapshot is written as the fallback row. */
+ /** The run is still live: the snapshot keeps the live shape. */
| 'live'
- /** A settling path owns the final row: the snapshot is withheld so it
- * cannot reopen the settled turn as an unfinished response. */
+ /** The terminal write failed and settled for a reconciliation frame: the
+ * snapshot is the turn's only row, so it persists with the terminal
+ * outcome and envelope. */
+ | 'terminal'
+ /** A durable terminal row exists: the snapshot is withheld so it cannot
+ * reopen the settled turn. */
| 'skip';
+/** The row flags a promoted compaction snapshot settles with, keyed by the
+ * reconciled claim's status: a stopped compaction is settled like the abort
+ * route writes it (nothing continues it), a completed run is a finished row,
+ * and everything else takes the error envelope. */
+export function resolveReconciledSnapshotEnvelope(status: unknown): {
+ unfinished: boolean;
+ error: boolean;
+} {
+ if (status === 'aborted' || status === 'complete') {
+ return { unfinished: false, error: false };
+ }
+ return { unfinished: false, error: true };
+}
+
/**
- * How the last-subscriber disconnect may persist this turn's snapshot, read
- * from the same-epoch job record the caller already loaded. A compaction
- * whose settling path owns the final row withholds the snapshot; the guard
- * reads the record that path writes, so the remaining window is the path's
- * own commit span. Ordinary turns keep writing their fallback row, settled
- * or not.
+ * How the last-subscriber disconnect may persist this turn's snapshot. An
+ * ordinary turn always keeps its live snapshot as the fallback row: its job
+ * record cannot prove a response row landed. A compaction whose settling path
+ * (completion, error, abort) durably owns the final row withholds the
+ * snapshot, so it cannot reopen the settled turn. A same-epoch compaction
+ * whose terminal claim settled for a reconciliation frame is promoted to the
+ * terminal row, but a reconciliation frame does not prove the row is missing
+ * (an error turn can land before it), so the response row is read first and a
+ * settled one withholds the promotion; the anchor must be persisted too, or
+ * the promotion would recreate the orphan an absent-anchor abort withheld.
*/
-export function resolveDisconnectSnapshotMode(
+export async function resolveDisconnectSnapshotMode(
isCompaction: boolean,
- jobRecord: { createdAt?: number; status?: string } | null | undefined,
- jobCreatedAt?: number,
-): DisconnectSnapshotMode {
+ jobRecord:
+ | {
+ createdAt?: number;
+ status?: string;
+ terminalPersistencePending?: boolean;
+ finalEvent?: string;
+ }
+ | null
+ | undefined,
+ jobCreatedAt: number | undefined,
+ {
+ anchorExists = async () => true,
+ responseRowSettled = async () => false,
+ }: {
+ anchorExists?: () => Promise<boolean>;
+ /** Whether a finished (not `unfinished`) response row already exists. */
+ responseRowSettled?: () => Promise<boolean>;
+ } = {},
+): Promise<DisconnectSnapshotMode> {
if (!isCompaction) {
return 'live';
}
- return isSettledJobRecord(jobRecord, jobCreatedAt) ? 'skip' : 'live';
+ if (isSettledJobRecord(jobRecord, jobCreatedAt)) {
+ return 'skip';
+ }
+ /** Another epoch's reconciliation frame belongs to a different
+ * generation and never promotes this one's snapshot. */
+ const sameEpoch = jobCreatedAt == null || jobRecord?.createdAt === jobCreatedAt;
+ if (!sameEpoch || !hasDurableReconcileFrame(jobRecord?.finalEvent)) {
+ return 'live';
+ }
+ /** Both reads are scoped to the turn's user and independent, so they run
+ * together; a settled row still wins over a persisted anchor. */
+ const [settled, anchored] = await Promise.all([responseRowSettled(), anchorExists()]);
+ return !settled && anchored ? 'terminal' : 'skip';
+}
+
+/**
+ * Every id a compaction's finished row can carry: the response id the stream
+ * preallocated, and the id the failed-turn path writes its error row under,
+ * derived from the anchor exactly as that path derives it. A settled row under
+ * either already ends the turn.
+ */
+function terminalRowIds(anchorMessageId: string, responseMessageId: string): string[] {
+ const errorRowId = getPreliminaryResponseMessageId({
+ compact: true,
+ messageId: anchorMessageId,
+ liveResponseMessageId: responseMessageId,
+ });
+ /** An anchor ending in one underscore derives itself; the failed-turn path
+ * then redirects its error row to the response id, so the anchor is never a
+ * terminal identity. */
+ return errorRowId == null || errorRowId === responseMessageId || errorRowId === anchorMessageId
+ ? [responseMessageId]
+ : [responseMessageId, errorRowId];
+}
+
+/** What the last-subscriber disconnect writes for this turn, if anything. */
+export type DisconnectSnapshotPlan =
+ | { write: false }
+ | {
+ write: true;
+ content: TMessageContentParts[];
+ unfinished: boolean;
+ error: boolean;
+ };
+
+/**
+ * Plans the last-subscriber disconnect snapshot through the caller's message
+ * reader: the mode, the marked content, and the row envelope. The caller only
+ * builds and writes the row, inside its own tenant context (the reads here
+ * run in it too). A live snapshot with nothing persistable is skipped; a
+ * promoted terminal snapshot synthesizes its outcome even from empty content.
+ */
+export async function planDisconnectSnapshot(
+ isCompaction: boolean,
+ {
+ content,
+ jobRecord,
+ jobCreatedAt,
+ userId,
+ conversationId,
+ anchorMessageId,
+ responseMessageId,
+ liveSaved = false,
+ getMessages,
+ onReadError,
+ }: {
+ /** The streamed content, already filtered to persistable parts. */
+ content: TMessageContentParts[];
+ jobRecord: Parameters<typeof resolveDisconnectSnapshotMode>[1];
+ jobCreatedAt: number | undefined;
+ userId: string;
+ conversationId: string;
+ anchorMessageId: string;
+ responseMessageId: string;
+ /** This process already wrote a live snapshot of the turn: only a terminal
+ * promotion writes again. */
+ liveSaved?: boolean;
+ getMessages: (
+ filter: { user: string; messageId: string; conversationId: string },
+ projection?: string,
+ ) => Promise<ReadableMessageRow[]>;
+ /** Receives a row read that failed while planning. */
+ onReadError?: (error: unknown) => void;
+ },
+): Promise<DisconnectSnapshotPlan> {
+ /** A failed row read proves nothing either way: it is reported, nothing is
+ * written, and the next disconnect plans again. */
+ let mode: DisconnectSnapshotMode;
+ try {
+ mode = await resolveDisconnectSnapshotMode(isCompaction, jobRecord, jobCreatedAt, {
+ anchorExists: async () =>
+ (await getMessages({ user: userId, messageId: anchorMessageId, conversationId }, '_id'))
+ .length > 0,
+ responseRowSettled: async () => {
+ const rows = await Promise.all(
+ terminalRowIds(anchorMessageId, responseMessageId).map((messageId) =>
+ getMessages({ user: userId, messageId, conversationId }, '_id unfinished'),
+ ),
+ );
+ return rows.some(([row]) => row != null && row.unfinished !== true);
+ },
+ });
+ } catch (error) {
+ onReadError?.(error);
+ return { write: false };
+ }
+ if (mode === 'skip' || (mode === 'live' && (liveSaved || content.length === 0))) {
+ return { write: false };
+ }
+ const marked = markAbortedCompactionContent(content, isCompaction, {
+ synthesizeFailure: mode === 'terminal',
+ });
+ if (marked.length === 0) {
+ return { write: false };
+ }
+ return {
+ write: true,
+ content: marked,
+ ...(mode === 'terminal'
+ ? resolveReconciledSnapshotEnvelope(jobRecord?.status)
+ : { unfinished: true, error: false }),
+ };
+}
+
+/**
+ * Stamps a promoted disconnect snapshot on its conversation once it is
+ * written: it is the turn's terminal row, so another device must learn the
+ * turn ended exactly as it would from any persisted terminal turn. A live
+ * snapshot is a fallback, not a reply, and stays unannounced. Never throws.
+ */
+export async function announceDisconnectSnapshot(
+ deps: ErrorTurnStampStore,
+ plan: DisconnectSnapshotPlan,
+ announcement: ErrorTurnAnnouncement,
+): Promise<void> {
+ if (!plan.write || plan.unfinished) {
+ return;
+ }
+ await announceErrorTurn(deps, announcement);
+}
+
+/**
+ * Runs a turn's disconnect snapshots one at a time and plans the snapshot once
+ * more when the turn settles, before its terminal cleanup releases the content
+ * and resume state the plan reads. Once every subscriber has left, no further
+ * disconnect arrives, so a terminal write that failed afterwards would leave a
+ * compaction's live snapshot (or nothing, when it left before any content)
+ * without its terminal outcome. Serializing keeps an earlier live write from
+ * landing over the promoted row.
+ */
+export function createDisconnectSnapshotQueue({
+ isCompaction,
+ persist,
+ readContent,
+ onSettleError,
+}: {
+ isCompaction: boolean;
+ /** Plans and writes one snapshot of the turn's content. */
+ persist: (content: TMessageContentParts[]) => Promise<void>;
+ /** The turn's stored content once it settled. */
+ readContent: () => Promise<TMessageContentParts[]>;
+ onSettleError?: (error: unknown) => void;
+}): {
+ onDisconnect: (content: TMessageContentParts[]) => Promise<void>;
+ settle: () => Promise<void>;
+} {
+ let pending: Promise<void> = Promise.resolve();
+ let disconnected = false;
+ const enqueue = (write: () => Promise<void>): Promise<void> => {
+ const next = pending.then(write);
+ pending = next.catch(() => undefined);
+ return next;
+ };
+ return {
+ onDisconnect: (content) => {
+ disconnected = true;
+ return enqueue(() => persist(content));
+ },
+ settle: async () => {
+ if (!isCompaction || !disconnected) {
+ return;
+ }
+ try {
+ await enqueue(async () => persist(await readContent()));
+ } catch (error) {
+ onSettleError?.(error);
+ }
+ },
+ };
}
/** How the abort route persists a stopped turn's prerequisite rows. */
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 56de9a96ea..1864758b48 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -4578,7 +4578,12 @@ class GenerationJobManagerClass {
streamId: string,
error?: string,
expectedCreatedAt?: number,
- options: { beforeErrorPublication?: () => Promise<void> } = {},
+ options: {
+ beforeErrorPublication?: () => Promise<void>;
+ /** Runs once the final event is stored and before the job's content and
+ * resume state are released; it must not throw. */
+ beforeTerminalCleanup?: () => Promise<void>;
+ } = {},
): Promise<boolean> {
const beforeErrorPublication = error ? options.beforeErrorPublication : undefined;
const claim = await this.claimTerminalJob(
@@ -4627,6 +4632,7 @@ class GenerationJobManagerClass {
}
}
+ await options.beforeTerminalCleanup?.();
await this.finishTerminalJob(claim);
return true;
}
diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.ts
index a595f76974..5ccb8ae587 100644
--- a/packages/api/src/stream/__tests__/startup.spec.ts
+++ b/packages/api/src/stream/__tests__/startup.spec.ts
@@ -3074,6 +3074,40 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.destroy();
});
+ it('runs the terminal cleanup hook after the reconciliation is stored and before cleanup', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = 'stream-error-cleanup-hook';
+ const job = await manager.createJob(streamId, 'user-1');
+ const finishTerminalJob = jest.spyOn(manager, 'finishTerminalJob');
+ let seenFinalEvent: string | undefined;
+ const beforeTerminalCleanup = jest.fn(async () => {
+ expect(finishTerminalJob).not.toHaveBeenCalled();
+ seenFinalEvent = (await jobStore.getJob(streamId))?.finalEvent;
+ });
+
+ await expect(
+ manager.completeJob(streamId, 'initialization failed', job.createdAt, {
+ beforeErrorPublication: async () => {
+ throw new Error('message store unavailable');
+ },
+ beforeTerminalCleanup,
+ }),
+ ).resolves.toBe(true);
+
+ expect(beforeTerminalCleanup).toHaveBeenCalledTimes(1);
+ expect(JSON.parse(seenFinalEvent ?? '{}')).toMatchObject({ reconcile: true });
+ expect(finishTerminalJob).toHaveBeenCalledTimes(1);
+ await manager.destroy();
+ });
+
it('atomically terminalizes a paused job when post-HITL persistence fails', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
const manager = new GenerationJobManagerClass();